/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2019-2022. All rights reserved.
 * Description: 集合通信hcom模块
 * Author: wangyahui
 * Create: 2019-11-28
 */

#include <algorithm>
#include <list>
#include <vector>
#include <string>
#include <securec.h>
#include <hccl/hccl_types.h>
#include "hcom_private.h"
#include "config.h"
#include "externalinput.h"
#include "workflow_pub.h"
#include "gradient_segment.h"
#include "hccl/base.h"
#include "rank_consistentcy_checker.h"
#include "param_check_pub.h"
#include "dlra_function.h"
#include "dltdt_function.h"
#include "../common/src/topo/topoinfo_detect.h"
#include "profiling_manager.h"
#include "../op_base/src/op_base.h"
#include "adapter_rts_common.h"
#include "adapter_prof.h"
#include "topoinfo_ranktableParser_pub.h"
#include "hccl/hcom.h"
#include "hccl_communicator.h"

using namespace std;
using namespace hccl;

HcclResult CallMsprofReportHostApi(hccl::hcclComm* hcclComm, HcclCMDType cmdType, uint64_t beginTime, u64 count,
    HcclDataType dataType)
{
    auto &profilingManager = hccl::ProfilingManager::Instance();
    AlgType algType;
    if(cmdType == HcclCMDType::HCCL_CMD_RECEIVE || cmdType == HcclCMDType::HCCL_CMD_SEND){
        algType.algoLevel0 = AlgTypeLevel0::ALG_LEVEL0_RESERVED;
        algType.algoLevel1 = AlgTypeLevel1::ALG_LEVEL1_RESERVED;
    } else {
        CHK_RET(hcclComm->GetAlgType(algType, cmdType));
    }
    uint64_t groupName = hrtMsprofGetHashId(hcclComm->GetIdentifier().c_str(), hcclComm->GetIdentifier().length());
    CHK_RET(profilingManager.CallMsprofReportHostApi(cmdType, beginTime, count, dataType, algType, groupName));
    HCCL_DEBUG("CallMsprofReportHostApi success, cmdType[%d], count[%llu], dataType[%d], algType[%d], groupName[%llu]",
        cmdType, count, dataType, algType.algoLevel0, groupName);
    return HCCL_SUCCESS;
}

HcclResult HcomCheckInitClusterInfo(const char *rankTableM, const char *identify);
HcclResult HcomFlushBackloggedGroups();
HcclResult HcomCollRemotePairedParaCheck(const HcomRemoteOperationParams &params);

HcclResult HcomInit(const char *rankTableM, const char *identify, WorkMode commWorkMode)
{
    HcclResult ret = HCCL_SUCCESS;
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    isAutoTuneModeOpen = false;
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();

    /*--------------入参合法性检测---------------------*/
    CHK_PTR_NULL(rankTableM);
    CHK_PTR_NULL(identify);

    /* 防止重复调用初始化 */
    CHK_PRT_RET((hcomInfo.pComm != nullptr|| hcomInfo.pCommBase != nullptr),
        HCCL_ERROR("[Init][Result]errNo[0x%016llx] identify[%s],\
        multiple initialization is not supported", HCOM_ERROR_CODE(HCCL_E_UNAVAIL), identify), HCCL_E_UNAVAIL);

    /* --------------初始化------------------------- */
    bool errorFlag = false;
    s32 logicDevId = 0;
    hcomInfo.params.commWorkMode = commWorkMode;
    do {
        ret = InitHcomMiscInfo(hcomInfo.params, rankTableM);
        CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] init other Info.",
            HCOM_ERROR_CODE(ret)), errorFlag = true);

        DevType deviceType;
        if (commWorkMode != HCCL_MODE_SCHED_OS) {
            CHK_PRT_BREAK(hrtGetDevice(&logicDevId) != HCCL_SUCCESS, , errorFlag = true);
            CHK_RET(hrtGetDeviceType(deviceType));
            // 为适配12包，做此修改
            (void)HcomCheckrtMemcpyAddrAsync();
        } else {
            deviceType = DevType::DEV_TYPE_NOSOC;
        }
        ret = CfgGetClusterInfo(rankTableM, identify, hcomInfo.params, hcomInfo.rankTable, deviceType);
        CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] cfg get ranktable[%p] info "\
            "error: identify[%s]", HCOM_ERROR_CODE(ret), rankTableM, identify), errorFlag = true);

        // HCCL_MODE_SCHED_OS仅在910A等同构集合通信场景才存在，此处仅希望异构资源池化场景进入
        if (commWorkMode != HCCL_MODE_SCHED_OS) {
            /*
            此case仅希望310 soc形态才能进入(此时有reacource.json来配置通信协议)
            但是310有板卡形态有可能进入(无reacource.json)，因此设定serverNum！=1的条件拦截
            因为310板卡形态跑大模型切分(allreduce)当前都是单机(serverNum=1)，因此无需通信协议解析
            */
            if (hcomInfo.rankTable.serverNum != SINGLE_SERVER_NUM &&
                (deviceType == DevType::DEV_TYPE_310P3 || deviceType == DevType::DEV_TYPE_310P1)) {
                CHK_RET(InitExternalInputHeterog());
            }
            // 调用初始化接口
            CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
        }

        const char *group;
        if (commWorkMode == HCCL_MODE_AI_CPU) {
            hcomInfo.pCommBase.reset((new (std::nothrow) hccl::HcclCommRpc(0, 0, HCCL_WORLD_GROUP)));
            CHK_PRT_RET(hcomInfo.pCommBase == nullptr, HCCL_ERROR("[Init][Result]hcomInfo.pCommBase is null,\
                create failed"), HCCL_E_PTR);
            ret = hcomInfo.pCommBase->init(hcomInfo.params, hcomInfo.rankTable);
            CHK_PRT_BREAK(ret != HCCL_SUCCESS,
                HCCL_ERROR("[Init][Result]errNo[0x%016llx] hcclComm init error",
                HCOM_ERROR_CODE(ret)), errorFlag = true);

            group = hcomInfo.pCommBase->GetIdentifier().c_str();
        } else if (commWorkMode == HCCL_MODE_SCHED_OS) {
            hcomInfo.pCommBase.reset((new (std::nothrow) hccl::HcclCommCpu(0, 0, HCCL_WORLD_GROUP)));
            CHK_PRT_RET(hcomInfo.pCommBase == nullptr, HCCL_ERROR("[Init][Result]hcomInfo.pCommBase is null,\
                create failed"), HCCL_E_PTR);
            ret = hcomInfo.pCommBase->init(hcomInfo.params, hcomInfo.rankTable);
            CHK_PRT_BREAK(ret != HCCL_SUCCESS,
                HCCL_ERROR("[Init][Result]errNo[0x%016llx] hcclComm init error",
                HCOM_ERROR_CODE(ret)), errorFlag = true);

            group = hcomInfo.pCommBase->GetIdentifier().c_str();
        } else {
            hcomInfo.pComm.reset(new (std::nothrow) hccl::hcclComm(0, 0, HCCL_WORLD_GROUP));

            CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Init][Result]hcomInfo.pComm is null,\
                create failed"), HCCL_E_PTR);
            ret = hcomInfo.pComm->init(hcomInfo.params, hcomInfo.rankTable);
            CHK_PRT_BREAK(ret != HCCL_SUCCESS,
                HCCL_ERROR("[Init][Result]errNo[0x%016llx] hcclComm init error",
                HCOM_ERROR_CODE(ret)), errorFlag = true);

            group = hcomInfo.pComm->GetIdentifier().c_str();
        }

        ret = ShowRanktableConfigInfo(hcomInfo.cloudFlag, hcomInfo.params, hcomInfo.rankTable);
        CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] put ranktable info error",
            HCOM_ERROR_CODE(ret)), errorFlag = true);
        if (commWorkMode != HCCL_MODE_SCHED_OS) {
            ret = InitWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB);
            CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] init work flow mode error",
                HCCL_ERROR_CODE(ret)), errorFlag = true);
        }

        ret = HcomFlushBackloggedGroups();
        CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] create backlogged group failed",
            HCOM_ERROR_CODE(ret)), errorFlag = true);

        if ((commWorkMode == HCCL_MODE_AI_CPU) ||
            (commWorkMode == HCCL_MODE_SCHED_OS)) {
            ret = HcomSetGroupTopoInfo(hcomInfo.pCommBase->GetIdentifier().c_str(), hcomInfo.rankTable.rankNum);
        } else {
            ret = HcomSetGroupTopoInfo(hcomInfo.pComm->GetIdentifier().c_str(), hcomInfo.rankTable.rankNum);
        }
        CHK_PRT_BREAK(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] SetGroupTopoInfo error, "\
            "group[%s]", HCOM_ERROR_CODE(ret), group), errorFlag = true);
    } while (0);

    if (errorFlag) {
        HCCL_ERROR("[Init][Result]hcom init failed, rankNum[%u], rank[%u], server[%s], device[%d], return[0x%016llx]",
            hcomInfo.rankTable.rankNum, hcomInfo.params.rank, hcomInfo.params.serverId.c_str(),
            logicDevId, HCOM_ERROR_CODE(ret));
        (void)HcomDestroy();
        return ret;
    }
    return HCCL_SUCCESS;
}

HcclResult HcomInitByString(const char *rankTableM, const char *identify, WorkMode commWorkMode)
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    CHK_PTR_NULL(rankTableM);
    CHK_PTR_NULL(identify);

    HcclUs startut = TIME_NOW();
    HcclResult ret = HCCL_SUCCESS;
    CHK_RET(InitExternalInput());
    CHK_RET(InitEnvConfig());
    CHK_RET(HcomCheckInitClusterInfo(rankTableM, identify));
    HCCL_RUN_INFO("Entry-HcomInitByString, identify[%s]", identify);

    ret = HcomInit(rankTableM, identify, commWorkMode);

    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[HcomInitByString]errNo[0x%016llx] rankTable[%p] identify[%s] "\
        "hcom init failed.", HCCL_ERROR_CODE(ret), rankTableM, identify), ret);
    hcomInfo.isHcomInit = true;

    /* 关键状态记录 */
    HCCL_RUN_INFO("[HCCL_TRACE]hcom init by string success,take time [%lld]us, rankTableAddr[%p], rankNum[%u], "\
        "rank[%u]", DURATION_US(TIME_NOW() - startut), rankTableM, hcomInfo.rankTable.rankNum, hcomInfo.params.rank);
    return HCCL_SUCCESS;
}

HcclResult GenerateRootInfo(HcclRootHandle &rootInfo)
{
    std::string identifier = "hccl_world_group";
    CHK_PRT_RET((identifier.length() >= ROOTINFO_INDENTIFIER_MAX_LENGTH),
        HCCL_ERROR("[Setup][Server]rootinfo identifier len[%zu] is invalid.", identifier.length()), HCCL_E_INTERNAL);
    s32 sret = memcpy_s(&rootInfo.identifier[0], sizeof(rootInfo.identifier), identifier.c_str(),
        (identifier.length() + 1));
    CHK_PRT_RET(sret != EOK, HCCL_ERROR("[Setup][Server]errNo[0x%016llx] memcpy failed. ret[%d], params:"\
        "destMaxSize[%zu],count[%zu]", HCOM_ERROR_CODE(HCCL_E_MEMORY), sret, sizeof(rootInfo.identifier),
        (identifier.length() + 1)), HCCL_E_MEMORY);

    s32 sRet = strncpy_s(rootInfo.ip, sizeof(rootInfo.ip), GetExternalInputMasterInfo().serverIp.GetReadableIP(),
        strlen(GetExternalInputMasterInfo().serverIp.GetReadableIP()));
    CHK_PRT_RET(sRet != EOK, HCCL_ERROR("[Setup][Server]str copy fail. return[%d]", sRet), HCCL_E_INTERNAL);

    rootInfo.port = GetExternalInputMasterInfo().port;
    rootInfo.nicDeploy = (GetExternalInputHcclDeviceNicDisable()) ?
        NICDeployment::NIC_DEPLOYMENT_HOST : NICDeployment::NIC_DEPLOYMENT_DEVICE;
    return HCCL_SUCCESS;
}

HcclResult HcomGenerteRanktable(std::string &rankTableM, std::string &rankId)
{
    CHK_RET(DlTdtFunction::GetInstance().DlTdtFunctionInit());
    CHK_RET(DlRaFunction::GetInstance().DlRaFunctionInit());
    s32 logicDevId = 0;
    u32 devPhyId = 0;
    CHK_RET(hrtGetDevice(&logicDevId));
    CHK_RET(hrtGetDevicePhyIdByIndex(static_cast<u32>(logicDevId), devPhyId));

    // true代表感知白名单disable配置
    CHK_RET(HcclNetInit(NICDeployment::NIC_DEPLOYMENT_HOST, devPhyId, logicDevId, true));

    HcclIpAddress localHostIp;
    CHK_RET(GetLocalHostIP(localHostIp, devPhyId));
    HcclRootHandle rootHandle;
    CHK_RET(GenerateRootInfo(rootHandle));

    std::shared_ptr<TopoInfoDetect> topoDetectAgent;
    EXECEPTION_CATCH(topoDetectAgent = std::make_shared<TopoInfoDetect>(), return HCCL_E_PTR);
    std::shared_ptr<TopoInfoDetect> topoDetectServer;
    EXECEPTION_CATCH(topoDetectServer = std::make_shared<TopoInfoDetect>(), return HCCL_E_PTR);

    bool retryEnable = GetExternalInputIntraServerRetryEnable() || GetExternalInputInterServerRetryEnable() ||
        GetExternalInputInterSuperPodRetryEnable();
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();

    bool isRoot = (localHostIp == GetExternalInputMasterInfo().serverIp &&
        logicDevId == static_cast<s32>(GetExternalInputMasterInfo().serverDeviceId));
    if (isRoot) {
        CHK_RET(topoDetectServer->SetupServerByMasterInfo(localHostIp, GetExternalInputMasterInfo().port, rootHandle));
    }

    CHK_PRT_RET(topoDetectAgent->SetupAgentByMasterInfo(localHostIp, rootHandle) != HCCL_SUCCESS,
        HCCL_ERROR("[Init][CommMasterInfo]setup topo detect error"), HCCL_E_INTERNAL);
    RankTable_t rankTable;
    CHK_PRT_RET(topoDetectAgent->GetCluterInfo(rankTable) != HCCL_SUCCESS,
        HCCL_ERROR("[Init][CommMasterInfo]GetCluterInfo error"), HCCL_E_INTERNAL);
    u32 rankIdNum = 0;
    CHK_PRT_RET(topoDetectAgent->GetRankId(rankIdNum) != HCCL_SUCCESS,
        HCCL_ERROR("[Init][CommMasterInfo]topoDetectAgent error"), HCCL_E_INTERNAL);

    CHK_RET(topoDetectAgent->TransformRankTableStr(rankTable, rankTableM));
    rankId = to_string(rankIdNum);
    CHK_PRT_RET(topoDetectAgent->WaitComplete(rootHandle) != HCCL_SUCCESS,
        HCCL_ERROR("[Init][CommMasterInfo]topoDetectAgent teardown fail"), HCCL_E_INTERNAL);

    CHK_PRT_RET(topoDetectAgent->GetAgentListenSocket(hcomInfo.params.commPortConfig) != HCCL_SUCCESS,
        HCCL_ERROR("[Init][CommMasterInfo]HcclGetCommListenSockets failed."), HCCL_E_INTERNAL);

    if (retryEnable) {
        hcomInfo.params.commConnections.isRoot = isRoot;
        if (isRoot) {
            hcomInfo.hcclCommTopoInfoDetectServer.insert({rootHandle.identifier, topoDetectServer});
            topoDetectServer->GetServerConnections(hcomInfo.params.commConnections.serverConnections);
        }
        hcomInfo.hcclCommTopoInfoDetectAgent.insert({rootHandle.identifier, topoDetectAgent});
        topoDetectAgent->GetAgentConnection(hcomInfo.params.commConnections.agentConnection);
    }

    return HCCL_SUCCESS;
}

HcclResult HcomInitByMasterInfo(const string &masterIp, const string &masterPort, const string & masterDeviceId,
    const string &rankSize, const string &rankIp)
{
    CHK_RET(SetMasterInfo(masterIp, masterPort, masterDeviceId, rankSize, rankIp));
    HcclUs startut = TIME_NOW();
    HcclResult ret = HCCL_SUCCESS;

    s32 logicDevId = 0;
    CHK_RET(hrtGetDevice(&logicDevId));
    // 读取rankTable文件到内存
    std::string rankTableM;
    std::string identify;
    HCCL_RUN_INFO("Entry-HcomInitByMasterInfo:masterIp[%s], masterPort[%s], master device id[%s], rankSize[%s], "
        "deviceId[%d]", masterIp.c_str(), masterPort.c_str(), masterDeviceId.c_str(), rankSize.c_str(), logicDevId);

    CHK_RET(InitExternalInput()); // 生成ranktable前需要提前感知部分配置
    CHK_RET(InitEnvConfig());
    ret = HcomGenerteRanktable(rankTableM, identify);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[HcomInitByMasterInfo]errNo[0x%016llx] masterIp[%s], masterPort[%s], "
        "masterDeviceId[%s] rankSize[%s] deviceId[%d] load rankTable error.", HCCL_ERROR_CODE(HCCL_E_INTERNAL),
        masterIp.c_str(), masterPort.c_str(),  masterDeviceId.c_str(), rankSize.c_str(), logicDevId), HCCL_E_INTERNAL);
    CHK_RET(HcomCheckInitClusterInfo(rankTableM.c_str(), identify.c_str()));

    // 调用初始化接口
    ret = HcomInit(rankTableM.c_str(), identify.c_str());
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[HcomInitByMasterInfo]errNo[0x%016llx] identify[%s] "
        "hcom init failed.", HCCL_ERROR_CODE(ret), identify.c_str()), ret);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    hcomInfo.isHcomInit = true;
    /* 关键状态记录 */
    HCCL_RUN_INFO("[HCCL_TRACE]hcom init by masterinfo success,take time [%lld]us, rankNum[%u], rank[%u], "\
        "server[%s], device[%d]", DURATION_US(TIME_NOW() - startut), hcomInfo.rankTable.rankNum,
        hcomInfo.params.rank, hcomInfo.params.serverId.c_str(), hcomInfo.params.logicDevId);
    return HCCL_SUCCESS;
}

HcclResult HcomSetProfilingMode(HcomProfilingMode profilingMode, string profilingOption)
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    HCCL_INFO("Set profiling option[%s].", profilingOption.c_str());
    hcomInfo.params.profilingMode = profilingMode;
    hcomInfo.params.profilingOption = profilingOption;
    return HCCL_SUCCESS;
}

HcclResult HcomDestroyOneDeviceHeterog(HcomInfo &hcomInfo)
{
    if (!hcomInfo.isHcomInit) {
        if (hcomInfo.pCommBase != nullptr) {
            hcomInfo.pCommBase = nullptr;
        }
        HCCL_INFO("[Destroy][Result]hcomInfo.isHcomInit is false.");

        /* 接口交互信息日志 */
        HCCL_INFO("Entry-HcomDestroy:void skip");
        return HCCL_SUCCESS;
    } else {
        if (hcomInfo.pCommBase != nullptr) {
            HcomUnSetGroupTopoInfo(hcomInfo.pCommBase->GetIdentifier().c_str());
        }
    }

    hcomInfo.pCommBase = nullptr;

    return HCCL_SUCCESS;
}

HcclResult HcomAllGather(const char *tag, void *inputPtr, void *outputPtr, u64 inputCount,
                         HcclDataType dataType, const char *group, rtStream_t stream)
{
    HcclResult ret;
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(inputCount == 0, HCCL_WARNING("input count is 0, return all gather success"), HCCL_SUCCESS);
    // 参数合法性校验

    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomAllGather", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomAllGather", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, inputCount, dataType, group, stream));

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    s32 streamId = 0;
    ret = hrtGetStreamId(stream, streamId);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllGather][Result]Call hrtGetStreamId error[%d].",
        ret), HCCL_E_RUNTIME);
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAllGather:tag[%s], inputPtr[%p], outputPtr[%p], inputCount[%llu], dataType[%s], "\
        "group[%s], streamId[%d], deviceLogicId[%d]", tag, inputPtr, outputPtr, inputCount, GetDataTypeEnumStr(dataType).c_str(),
        strGroup.c_str(), streamId, deviceLogicId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));

    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, inputCount, inputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, rankId);

    ret = hcclComm->AllGather(tag, inputPtr, outputPtr, inputCount, dataType, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllGather][Result]errNo[0x%016llx] hcclComm all gather error, "\
        "tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s]", HCOM_ERROR_CODE(ret),
        tag, inputPtr, outputPtr, inputCount, GetDataTypeEnumStr(dataType).c_str()), ret);

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_ALLGATHER, beginTime, inputCount, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom allgather success,take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], "\
        "count[%llu], data_type[%s]", DURATION_US(TIME_NOW() - startut), tag, inputPtr, outputPtr,
        inputCount, GetDataTypeEnumStr(dataType).c_str());

    return HCCL_SUCCESS;
}

HcclResult HcomAllGatherV(const char *tag, const void *sendBuf, u64 sendCount, const void *recvBuf,
    const void *recvCounts, const void *rdispls, HcclDataType dataType, const char *group, rtStream_t stream)
{
    HcclResult ret;
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    // 参数合法性校验
    CHK_PTR_NULL(recvCounts);
    CHK_PTR_NULL(rdispls);
    RPT_INPUT_ERR(sendBuf == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomAllGatherV", "sendBuf", "nullptr", "please check sendBuf"}));
    CHK_PTR_NULL(sendBuf);
    RPT_INPUT_ERR(recvBuf == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomAllGatherV", "recvBuf", "nullptr", "please check recvBuf"}));
    CHK_PTR_NULL(recvBuf);
    CHK_RET(HcomCheckOpParam(tag, 0, dataType, group, stream));

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    s32 streamId = 0;
    ret = hrtGetStreamId(stream, streamId);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllGatherV][Result]Call hrtGetStreamId error[%d].",
        ret), HCCL_E_RUNTIME);
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAllGatherV:tag[%s], inputPtr[%p], outputPtr[%p], sendCount[%llu], dataType[%s], "\
        "recvCounts[%p], rdispls[%p], group[%s], streamId[%d], deviceLogicId[%d]", tag, sendBuf, recvBuf, sendCount,
        GetDataTypeEnumStr(dataType).c_str(), recvCounts, rdispls, strGroup.c_str(), streamId, deviceLogicId);
    CHK_RET(PrintMemoryAttr(sendBuf));
    CHK_RET(PrintMemoryAttr(recvBuf));

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));

    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, sendCount, sendBuf, recvBuf, dataType, INVALID_VALUE_RANKID, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, rankId);

    ret = hcclComm->AllGatherV(tag, sendBuf, sendCount, recvBuf, recvCounts, rdispls, dataType, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllGatherV][Result]errNo[0x%016llx] hcclComm all gatherv error, "\
        "tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s]", HCOM_ERROR_CODE(ret),
        tag, sendBuf, recvBuf, sendCount, GetDataTypeEnumStr(dataType).c_str()), ret);

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_ALLGATHER_V, beginTime, sendCount, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom allgatherv success,take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], "\
        "count[%llu], data_type[%s]", DURATION_US(TIME_NOW() - startut), tag, sendBuf, recvBuf,
        sendCount, GetDataTypeEnumStr(dataType).c_str());

    return HCCL_SUCCESS;
}

HcclResult HcomGetInitStatus(bool& initiated)
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    initiated = !(hcomInfo.pComm == nullptr);

    HCCL_INFO("Get Hcom Init Status: [%d]", initiated);
    return HCCL_SUCCESS;
}
#ifndef CCL_KERNEL_AICPU
HcclResult HcomAllReduce(const char *tag, void *inputPtr, void *outputPtr, u64 count, HcclDataType dataType,
                         HcclReduceOp op, const char *group, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return all reduce success"), HCCL_SUCCESS);
    // 入参合法性校验

    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomAllReduce", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomAllReduce", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);

    s32 streamId = 0;
    CHK_RET(HcomCheckReductionOp(op));
    CHK_RET(HcomCheckOpParam(tag, count, dataType, group, stream));
    CHK_RET(hrtGetStreamId(stream, streamId));

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAllReduce:tag[%s], inputPtr[%p], outputPtr[%p], count[%llu], dataType[%s], op[%s], "\
        "group[%s], streamId[%d], deviceLogicId[%d]",
        tag, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(),
        strGroup.c_str(), streamId, deviceLogicId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, strGroup, op);
    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, rankId);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->AllReduce(tag, inputPtr, outputPtr, count, dataType, op, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllReduce][Result]errNo[0x%016llx] hcclComm all reduce error, "\
        "tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s], op[%s]", HCOM_ERROR_CODE(ret), tag,
        inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str()), ret);

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_ALLREDUCE, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom allreduce success,take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], "\
        "count[%llu], data_type[%s], op[%s]", DURATION_US(TIME_NOW() - startut), tag, inputPtr,
        outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());

    return HCCL_SUCCESS;
}
#endif
HcclResult HcomBroadcast(const char *tag, void *ptr, u64 count, HcclDataType dataType,
    u32 root, const char *group, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return broadcast success"), HCCL_SUCCESS);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    // 入参合法性校验

    RPT_INPUT_ERR(ptr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomBroadcast", "ptr", "nullptr", "please check ptr"}));
    CHK_PTR_NULL(ptr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, group, stream));
    CHK_RET(HcomCheckUserRank(hcomInfo.params.totalRanks, root));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomBroadcast:tag[%s], ptr[%p], count[%llu], dataType[%s], root[%u], "\
        "group[%s], streamId[%d], deviceLogicId[%d]", tag, ptr, count, GetDataTypeEnumStr(dataType).c_str(), root, strGroup.c_str(),
        streamId, deviceLogicId);

    CHK_RET(PrintMemoryAttr(ptr));

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    /* 记录指令信息用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_BROADCAST, tag, count, dataType,
        root, hcclComm->GetConfigInCCLbufferSize(), 0, strGroup.c_str(), hcclComm->GetRankTableCrc()));
    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, ptr, ptr, dataType, root, strGroup, HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, hcomInfo.params.totalRanks, hcomInfo.params.userRank);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->Broadcast(tag, ptr, count, dataType, root, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS,
        HCCL_ERROR("[Broadcast][Result]errNo[0x%016llx] hcclComm broadcast error,tag[%s], input_ptr[%p],"
        "count[%llu], data_type[%s], root[%u]", HCOM_ERROR_CODE(ret), tag, ptr, count,
        GetDataTypeEnumStr(dataType).c_str(), root), ret);
    CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(tag));
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_BROADCAST, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom broadcast success,take time [%lld]us,tag[%s], input_ptr[%p], count[%llu], data_type[%s], "\
        "root[%u]", DURATION_US(TIME_NOW() - startut), tag, ptr, count, GetDataTypeEnumStr(dataType).c_str(), root);

    return HCCL_SUCCESS;
}

HcclResult HcomReduce(const char *tag, void *inputPtr, void *outputPtr, u64 count, HcclDataType dataType,
    HcclReduceOp op, u32 root, const char *group, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return reduce success"), HCCL_SUCCESS);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    // 入参合法性校验
    CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Reduce][Result]hcomInfo.pComm is null, "\
        "please check if the initialize process is called."), HCCL_E_PTR);
    RPT_INPUT_ERR(tag == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReduce", "tag", "nullptr", "please check tag"}));
    CHK_PTR_NULL(tag);
    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReduce", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReduce", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, group, stream));
    CHK_RET(HcomCheckReductionOp(op));
    CHK_RET(HcomCheckUserRank(hcomInfo.params.totalRanks, root));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomReduce:tag[%s], inputPtr[%p], outputPtr[%p], count[%llu], dataType[%s], op[%s], "
        "root[%u], group[%s], streamId[%d], deviceLogicId[%d]",
        tag, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), root,
        strGroup.c_str(), streamId, deviceLogicId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(group, hcclComm));
    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, outputPtr, dataType, root, strGroup, op);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, hcomInfo.params.totalRanks, hcomInfo.params.userRank);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->Reduce(tag, inputPtr, outputPtr, count, dataType, op, root, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Reduce][Result]errNo[0x%016llx] hcclComm reduce error, tag[%s], "\
        "input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s], op[%s], root[%u]", HCOM_ERROR_CODE(ret), tag,
        inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), root), ret);
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_REDUCE, beginTime, count, dataType));
    HcclUs endut = TIME_NOW();
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom reduce success, take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], "\
        "data_type[%s], op[%s], root[%u]", DURATION_US(endut - startut), tag, inputPtr, outputPtr, count,
        GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), root);

    return HCCL_SUCCESS;
}

HcclResult HcomReduceScatter(const char *tag, void *inputPtr, void *outputPtr, u64 count,
    HcclDataType dataType, HcclReduceOp op, const char *group, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return reduce scatter success"), HCCL_SUCCESS);
    // 入参合法性校验

    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReduceScatter", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReduceScatter", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, group, stream));
    CHK_RET(HcomCheckReductionOp(op));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomReduceScatter:tag[%s], inputPtr[%p], outputPtr[%p], count[%llu], dataType[%s], op[%s], "\
        "group[%s], streamId[%d], deviceLogicId[%d]", tag, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(),
        GetReduceOpEnumStr(op).c_str(), strGroup.c_str(), streamId, deviceLogicId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));

    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, strGroup, op);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, rankId);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->ReduceScatter(tag, inputPtr, outputPtr, count, dataType, op, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[ReduceScatter][Result]errNo[0x%016llx] hcclComm reduce_scatter "\
        "error, tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s], op[%s]", HCOM_ERROR_CODE(ret),
        tag, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());, ret);

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_REDUCE_SCATTER, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO(
        "hcom reduceScatter success, take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], "\
        "data_type[%s], op[%s]", DURATION_US(TIME_NOW() - startut), tag, inputPtr, outputPtr, count,
        GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());

    return HCCL_SUCCESS;
}

HcclResult HcomReduceScatterV(const char *tag, void *sendBuf, const void *sendCounts, const void *sdispls, void *recvBuf,
    u64 recvCount, HcclDataType dataType, HcclReduceOp op, const char *group, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    // 入参合法性校验
    CHK_PTR_NULL(sendCounts);
    CHK_PTR_NULL(sdispls);
    RPT_INPUT_ERR(sendBuf == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReduceScatterV", "sendBuf", "nullptr", "please check sendBuf"}));
    CHK_PTR_NULL(sendBuf);
    RPT_INPUT_ERR(recvBuf == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReduceScatterV", "recvBuf", "nullptr", "please check recvBuf"}));
    CHK_PTR_NULL(recvBuf);
    CHK_RET(HcomCheckOpParam(tag, 0, dataType, group, stream));
    CHK_RET(HcomCheckReductionOp(op));
 
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcomReduceScatterV:tag[%s], inputPtr[%p], outputPtr[%p], count[%llu], dataType[%s], op[%s], "\
        "group[%s], streamId[%d]", tag, sendBuf, recvBuf, recvCount, GetDataTypeEnumStr(dataType).c_str(),
        GetReduceOpEnumStr(op).c_str(), strGroup.c_str(), streamId);
    CHK_RET(PrintMemoryAttr(sendBuf));
    CHK_RET(PrintMemoryAttr(recvBuf));
 
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
 
    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, 0, sendBuf, recvBuf, dataType, INVALID_VALUE_RANKID, strGroup, op);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, rankId);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->ReduceScatterV(tag, sendBuf, sendCounts, sdispls, recvBuf, recvCount, dataType, op, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[ReduceScatterV][Result]errNo[0x%016llx] hcclComm reduce_scatter "\
        "error, tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s], op[%s]", HCOM_ERROR_CODE(ret),
        tag, sendBuf, recvBuf, recvCount, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());, ret);
 
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_REDUCE_SCATTER, beginTime, 0, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO(
        "hcom reducescatterv success, take time [%lld]us, tag[%s], sendBuf[%p], recvBuf[%p], count[%llu], "\
        "data_type[%s], op[%s]", DURATION_US(TIME_NOW() - startut), tag, sendBuf, recvBuf, recvCount,
        GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());
 
    return HCCL_SUCCESS;
}

/*
 * 点对点发送接口: 需要对应的hcom_receive执行才会实际发送。先分片，条件满足之后改为不分片
 * 发送端需要接收端准备好才会发送
 */
HcclResult HcomSend(const char *tag, void *inputPtr, u64 count, HcclDataType dataType, u32 destRank,
    u32 srTag, const char *group, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return send success"), HCCL_SUCCESS);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    // 入参合法性校验

    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomSend", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, group, stream));
    CHK_RET(HcomCheckUserRank(hcomInfo.params.totalRanks, destRank));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomSend:tag[%s], inputPtr[%p], count[%llu], dataType[%s], destRank[%u], srTag[%u], "\
        "group[%s], streamId[%d], deviceLogicId[%d]", tag, inputPtr,  count, GetDataTypeEnumStr(dataType).c_str(), destRank, srTag,
        strGroup.c_str(), streamId, deviceLogicId);

    CHK_RET(PrintMemoryAttr(inputPtr));
    u32 destWorldRank = INVALID_VALUE_RANKID;
    CHK_RET(HcomGetWorldRankFromGroupRank(strGroup.c_str(), destRank, &destWorldRank));

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));

    u32 localGroupRank = INVALID_VALUE_RANKID;
    CHK_RET(HcomGetGroupRankFromWorldRank(hcomInfo.params.rank, strGroup.c_str(), &localGroupRank));
    /* 记录指令信息用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_SEND, tag, count, dataType,
        destRank, srTag, localGroupRank, hcclComm->GetConfigInCCLbufferSize(), 0, strGroup.c_str(),
        hcclComm->GetRankTableCrc()));

    HCCL_PROFILER_ADD_TAG_SENDRECV(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, inputPtr, dataType, INVALID_VALUE_RANKID, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK_SENDRECV(strGroup, hcomInfo.params.totalRanks, localGroupRank, destRank);
    /* 调用HCCL的send, 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->send(tag, inputPtr, count, dataType, destRank, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Send][Result]errNo[0x%016llx] hcclComm send error, tag[%s], "\
        "inputPtr[%p], count[%llu], dataType[%s], destRank[%u], group[%s]", HCOM_ERROR_CODE(ret), tag,
        inputPtr, count, GetDataTypeEnumStr(dataType).c_str(), destRank, strGroup.c_str()), ret);
    CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(tag));

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_SEND, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom send success,time[%lld]us,tag[%s],inputPtr[%p],count[%llu],dataType[%s],destRank[%u],"
        "srTag[%u]",
        DURATION_US(TIME_NOW() - startut), tag, inputPtr, count, GetDataTypeEnumStr(dataType).c_str(), destRank, srTag);

    return HCCL_SUCCESS;
}

/*
 * 点对点接收接口: 需要对应的hcom_receive执行才会实际发送。先分片，条件满足之后改为不分片
 * 发送端需要接收端准备好才会发送
 */
HcclResult HcomReceive(const char *tag, void *outputPtr, u64 count, HcclDataType dataType, u32 srcRank,
    u32 srTag, const char *group, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return receive success"), HCCL_SUCCESS);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    // 入参合法性校验

    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomReceive", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, group, stream));
    CHK_RET(HcomCheckUserRank(hcomInfo.params.totalRanks, srcRank));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomReceive:tag[%s], outputPtr[%p], count[%llu], dataType[%s], srcRank[%u], srTag[%u],"\
        "group[%s], streamId[%d], deviceLogicId[%d]", tag, outputPtr,  count, GetDataTypeEnumStr(dataType).c_str(), srcRank, srTag,
        strGroup.c_str(), streamId, deviceLogicId);
    CHK_RET(PrintMemoryAttr(outputPtr));

    u32 srcWorldRank = INVALID_VALUE_RANKID;
    CHK_RET(HcomGetWorldRankFromGroupRank(strGroup.c_str(), srcRank, &srcWorldRank));

    std::shared_ptr<hccl::hcclComm>  hcclComm;
    HcclResult ret = HcomGetCommByGroup(strGroup.c_str(), hcclComm);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Receive][Result]errNo[0x%016llx] get comm by group[%s] error",
        HCOM_ERROR_CODE(ret), strGroup.c_str()), ret);

    u32 localGroupRank = INVALID_VALUE_RANKID;
    CHK_RET(HcomGetGroupRankFromWorldRank(hcomInfo.params.rank, strGroup.c_str(), &localGroupRank));
    /* 记录指令信息用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_RECEIVE, tag, count, dataType,
        srcRank, srTag, localGroupRank, hcclComm->GetConfigOutCCLbufferSize(), 0,
        strGroup.c_str(), hcclComm->GetRankTableCrc()));

    HCCL_PROFILER_ADD_TAG_SENDRECV(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, outputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK_SENDRECV(strGroup, hcomInfo.params.totalRanks, localGroupRank, srcRank);
    ret = hcclComm->receive(tag, outputPtr, count, dataType, srcRank, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Receive][Result]errNo[0x%016llx] hcclComm receive error,tag[%s], "\
        "outputPtr[%p], count[%llu], dataType[%s], srcRank[%u], group[%s]", HCOM_ERROR_CODE(ret), tag,
        outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), srcRank, strGroup.c_str()), ret);

    CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(tag));
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_RECEIVE, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom receive success,time[%lld]us,tag[%s],outputPtr[%p],count[%llu],dataType[%s],srcRank[%u],"
        "srTag[%u]",
        DURATION_US(TIME_NOW() - startut), tag, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), srcRank, srTag);

    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphAllGather(const char *tag, void *inputPtr, void *outputPtr, u64 inputCount,
    HcclDataType dataType, s64 opBaseHcom, rtStream_t stream)
{
    HcclResult ret;
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    CHK_PRT_RET(inputCount == 0, HCCL_WARNING("input count is 0, return all gather success"), HCCL_SUCCESS);
    // 参数合法性校验
    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphAllGather", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphAllGather", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, inputCount, dataType, stream));

    s32 streamId = 0;
    ret = hrtGetStreamId(stream, streamId);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllGather][Result]Call hrtGetStreamId error[%d].",
        ret), HCCL_E_RUNTIME);
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcclCommGraphAllGather:tag[%s], inputPtr[%p], outputPtr[%p], inputCount[%llu], dataType[%s], "\
        "opBaseHcom[%lld] streamId[%d]", tag, inputPtr, outputPtr, inputCount, GetDataTypeEnumStr(dataType).c_str(),
        opBaseHcom, streamId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    HCCL_PROFILER_ADD_TAG(tag, hcclComm->GetIdentifier(), GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());

    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_OPDATA_OP(tag, inputCount, inputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, \
        hcclComm->GetIdentifier(), HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(hcclComm->GetIdentifier(), rankSize, rankId);

    ret = hcclComm->AllGather(tag, inputPtr, outputPtr, inputCount, dataType, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllGather][Result]errNo[0x%016llx] hcclComm all gather error, "\
        "tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s]", HCOM_ERROR_CODE(ret),
        tag, inputPtr, outputPtr, inputCount, GetDataTypeEnumStr(dataType).c_str()), ret);
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_ALLGATHER, beginTime, inputCount, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("HcclCommGraphAllGather success,take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], "\
        "count[%llu], data_type[%s], opBaseHcom[%lld]", DURATION_US(TIME_NOW() - startut), tag,
        inputPtr, outputPtr, inputCount, GetDataTypeEnumStr(dataType).c_str(), opBaseHcom);

    return HCCL_SUCCESS;
}

#ifndef CCL_KERNEL_AICPU
HcclResult HcclCommGraphAllReduce(const char *tag, void *inputPtr, void *outputPtr, u64 count,
    HcclDataType dataType, HcclReduceOp op, s64 opBaseHcom, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return all reduce success"), HCCL_SUCCESS);
    // 入参合法性校验
    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphAllReduce", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphAllReduce", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);

    CHK_RET(HcomCheckReductionOp(op));
    CHK_RET(HcomCheckOpParam(tag, count, dataType, stream));
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));

    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcclCommGraphAllReduce:tag[%s], inputPtr[%p], outputPtr[%p], count[%llu], dataType[%s], "\
        "op[%s], opBaseHcom[%lld], streamId[%d]", tag, inputPtr, outputPtr, count,
        GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), opBaseHcom, streamId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    HCCL_PROFILER_ADD_TAG(tag, hcclComm->GetIdentifier(), GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, \
        hcclComm->GetIdentifier(), op);

    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_GROUPRANK(hcclComm->GetIdentifier(), rankSize, rankId);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->AllReduce(tag, inputPtr, outputPtr, count, dataType, op, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[AllReduce][Result]errNo[0x%016llx] hcclComm all reduce error, "\
        "tag[%s], input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s], op[%s]", HCOM_ERROR_CODE(ret), tag,
        inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str()), ret);
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_ALLREDUCE, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("HcclCommGraphAllReduce success,take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], "\
        "count[%llu], data_type[%s], op[%s], opBaseHcom[%lld]", DURATION_US(TIME_NOW() - startut), tag, inputPtr,
        outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), opBaseHcom);

    return HCCL_SUCCESS;
}
#endif
HcclResult HcclCommGraphReduce(const char *tag, void *inputPtr, void *outputPtr, u64 count, HcclDataType dataType,
    HcclReduceOp op, u32 root, s64 opBaseHcom, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return reduce success"), HCCL_SUCCESS);

    // 入参合法性校验
    RPT_INPUT_ERR(tag == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphReduce", "tag", "nullptr", "please check tag"}));
    CHK_PTR_NULL(tag);
    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphReduce", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphReduce", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, stream));
    CHK_RET(HcomCheckReductionOp(op));
    u32 totalRanks = 0;
    CHK_RET(HcclCommGraphGetRankSize(opBaseHcom, &totalRanks));
    CHK_RET(HcomCheckUserRank(totalRanks, root));
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcclCommGraphReduce:tag[%s], inputPtr[%p], outputPtr[%p], count[%llu], dataType[%s], op[%s], "
        "root[%u], opBaseHcom[%lld], streamId[%d]",
        tag, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), root,
        opBaseHcom, streamId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    HCCL_PROFILER_ADD_TAG(tag, hcclComm->GetIdentifier(), GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, outputPtr, dataType, root, hcclComm->GetIdentifier(), op);
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_GROUPRANK(hcclComm->GetIdentifier(), totalRanks, rankId);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->Reduce(tag, inputPtr, outputPtr, count, dataType, op, root, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Reduce][Result]errNo[0x%016llx] hcclComm reduce error, tag[%s], "\
        "input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s], op[%s], root[%u]", HCOM_ERROR_CODE(ret), tag,
        inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), root), ret);
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_REDUCE, beginTime, count, dataType));
    HcclUs endut = TIME_NOW();
    /* 关键状态记录 */
    HCCL_RUN_INFO("HcclCommGraphReduce success, take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], "
        "count[%llu], data_type[%s], op[%s], root[%u], opBaseHcom[%lld]",
        DURATION_US(endut - startut), tag, inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(),
        GetReduceOpEnumStr(op).c_str(), root, opBaseHcom);

    return HCCL_SUCCESS;
}
HcclResult HcclCommGraphBroadcast(const char *tag, void *ptr, u64 count, HcclDataType dataType, u32 root,
    s64 opBaseHcom, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return broadcast success"), HCCL_SUCCESS);

    // 入参合法性校验
    RPT_INPUT_ERR(ptr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphBroadcast", "ptr", "nullptr", "please check ptr"}));
    CHK_PTR_NULL(ptr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, stream));
    u32 totalRanks = 0;
    CHK_RET(HcclCommGraphGetRankSize(opBaseHcom, &totalRanks));
    CHK_RET(HcomCheckUserRank(totalRanks, root));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcclCommGraphBroadcast:tag[%s], ptr[%p], count[%llu], dataType[%s], root[%u], "\
        "opBaseHcom[%lld], streamId[%d]", tag, ptr, count, GetDataTypeEnumStr(dataType).c_str(), root, opBaseHcom,
        streamId);

    CHK_RET(PrintMemoryAttr(ptr));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    /* 记录指令信息用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_BROADCAST, tag, count, dataType,
        root, hcclComm->GetConfigInCCLbufferSize(), 0, hcclComm->GetIdentifier().c_str(), hcclComm->GetRankTableCrc()));
    HCCL_PROFILER_ADD_TAG(tag, hcclComm->GetIdentifier(), GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, ptr, ptr, dataType, root, hcclComm->GetIdentifier(), \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_GROUPRANK(hcclComm->GetIdentifier(), totalRanks, rankId);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->Broadcast(tag, ptr, count, dataType, root, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS,
        HCCL_ERROR("[Broadcast][Result]errNo[0x%016llx] hcclComm broadcast error,tag[%s], input_ptr[%p],"
        "count[%llu], data_type[%s], root[%u]", HCOM_ERROR_CODE(ret), tag, ptr, count,
        GetDataTypeEnumStr(dataType).c_str(), root), ret);
    CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(tag));
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_BROADCAST, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("HcclCommGraphBroadcast success,take time [%lld]us,tag[%s], input_ptr[%p], count[%llu], "\
        "data_type[%s], root[%u], opBaseHcom[%lld]", DURATION_US(TIME_NOW() - startut), tag, ptr, count,
        GetDataTypeEnumStr(dataType).c_str(), root, opBaseHcom);

    return HCCL_SUCCESS;
}
HcclResult HcclCommGraphReduceScatter(const char *tag, void *inputPtr, void *outputPtr, u64 count,
    HcclDataType dataType, HcclReduceOp op, s64 opBaseHcom, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return reduce scatter success"), HCCL_SUCCESS);
    // 入参合法性校验
    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphReduceScatter", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphReduceScatter", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, stream));
    CHK_RET(HcomCheckReductionOp(op));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcclCommGraphReduceScatter:tag[%s], inputPtr[%p], outputPtr[%p], count[%llu], dataType[%s], "\
        "op[%s], opBaseHcom[%lld], streamId[%d]", tag, inputPtr, outputPtr, count,
        GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), opBaseHcom, streamId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    CHK_RET(PrintMemoryAttr(outputPtr));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));

    u32 rankSize = INVALID_VALUE_RANKSIZE;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = INVALID_VALUE_RANKID;
    CHK_RET(hcclComm->GetGroupRank(rankId));
    HCCL_PROFILER_ADD_TAG(tag, hcclComm->GetIdentifier(), GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, \
        hcclComm->GetIdentifier(), op);
    HCCL_PROFILER_ADD_GROUPRANK(hcclComm->GetIdentifier(), rankSize, rankId);
    /* 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->ReduceScatter(tag, inputPtr, outputPtr, count, dataType, op, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS,
        HCCL_ERROR("[ReduceScatter][Result]errNo[0x%016llx] hcclComm reduce_scatter error, tag[%s],"
        "input_ptr[%p], output_ptr[%p], count[%llu], data_type[%s], op[%s]", HCOM_ERROR_CODE(ret), tag,
        inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str());, ret);

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_REDUCE_SCATTER, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO(
        "HcclCommGraphReduceScatter success, take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], "\
        "count[%llu], data_type[%s], op[%s], opBaseHcom[%lld]", DURATION_US(TIME_NOW() - startut), tag,
        inputPtr, outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), GetReduceOpEnumStr(op).c_str(), opBaseHcom);

    return HCCL_SUCCESS;
}
HcclResult HcclCommGraphSend(const char *tag, void *inputPtr, u64 count, HcclDataType dataType,
    u32 destRank, u32 srTag, s64 opBaseHcom, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return send success"), HCCL_SUCCESS);

    // 入参合法性校验
    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphSend", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, stream));
    u32 totalRanks = 0;
    CHK_RET(HcclCommGraphGetRankSize(opBaseHcom, &totalRanks));
    CHK_RET(HcomCheckUserRank(totalRanks, destRank));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcclCommGraphSend:tag[%s], inputPtr[%p], count[%llu], dataType[%s], destRank[%u], srTag[%u], "\
        "opBaseHcom[%lld], streamId[%d]", tag, inputPtr,  count, GetDataTypeEnumStr(dataType).c_str(), destRank,
        srTag, opBaseHcom, streamId);

    CHK_RET(PrintMemoryAttr(inputPtr));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));

    u32 rankID = 0;
    CHK_RET(HcclCommGraphGetRankId(opBaseHcom, &rankID));
    /* 记录指令信息用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_SEND, tag, count, dataType,
        destRank, srTag, rankID, hcclComm->GetConfigInCCLbufferSize(), 0,
        hcclComm->GetIdentifier().c_str(), hcclComm->GetRankTableCrc()));

    HCCL_PROFILER_ADD_TAG_SENDRECV(tag, hcclComm->GetIdentifier(), GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, inputPtr, inputPtr, dataType, INVALID_VALUE_RANKID, \
        hcclComm->GetIdentifier(), HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK_SENDRECV(hcclComm->GetIdentifier(), totalRanks, rankID, destRank);

    /* 调用HCCL的send, 入参的正确性由HCCL确保 */
    HcclResult ret = hcclComm->send(tag, inputPtr, count, dataType, destRank, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Send][Result]errNo[0x%016llx] hcclComm send error, tag[%s], "\
        "inputPtr[%p], count[%llu], dataType[%s], destRank[%u]", HCOM_ERROR_CODE(ret), tag,
        inputPtr, count, GetDataTypeEnumStr(dataType).c_str(), destRank), ret);
    CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(tag));

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_SEND, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("HcclCommGraphSend success,time[%lld]us,tag[%s],inputPtr[%p],count[%llu],dataType[%s],destRank[%u],"\
        "srTag[%u], opBaseHcom[%lld]", DURATION_US(TIME_NOW() - startut), tag, inputPtr, count,
        GetDataTypeEnumStr(dataType).c_str(), destRank, srTag, opBaseHcom);

    return HCCL_SUCCESS;
}
HcclResult HcclCommGraphReceive(const char *tag, void *outputPtr, u64 count, HcclDataType dataType,
    u32 srcRank, u32 srTag, s64 opBaseHcom, rtStream_t stream)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    CHK_PRT_RET(count == 0, HCCL_WARNING("input count is 0, return receive success"), HCCL_SUCCESS);
    // 入参合法性校验
    RPT_INPUT_ERR(outputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphReceive", "outputPtr", "nullptr", "please check outputPtr"}));
    CHK_PTR_NULL(outputPtr);
    CHK_RET(HcomCheckOpParam(tag, count, dataType, stream));
    u32 totalRanks = 0;
    CHK_RET(HcclCommGraphGetRankSize(opBaseHcom, &totalRanks));
    CHK_RET(HcomCheckUserRank(totalRanks, srcRank));

    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcclCommGraphReceive:tag[%s], outputPtr[%p], count[%llu], dataType[%s], srcRank[%u], "
        "srTag[%u], opBaseHcom[%lld], streamId[%d]", tag, outputPtr,  count, GetDataTypeEnumStr(dataType).c_str(),
        srcRank, srTag, opBaseHcom, streamId);
    CHK_RET(PrintMemoryAttr(outputPtr));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));

    /* 记录指令信息用于一致性校验 */
    u32 rankID = 0;
    CHK_RET(HcclCommGraphGetRankId(opBaseHcom, &rankID));
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_RECEIVE, tag, count, dataType,
        srcRank, srTag, rankID, hcclComm->GetConfigOutCCLbufferSize(), 0,
        hcclComm->GetIdentifier().c_str(), hcclComm->GetRankTableCrc()));

    HCCL_PROFILER_ADD_TAG_SENDRECV(tag, hcclComm->GetIdentifier(), GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, count, outputPtr, outputPtr, dataType, INVALID_VALUE_RANKID, \
        hcclComm->GetIdentifier(), HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK_SENDRECV(hcclComm->GetIdentifier(), totalRanks, rankID, srcRank);

    HcclResult ret = hcclComm->receive(tag, outputPtr, count, dataType, srcRank, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Receive][Result]errNo[0x%016llx] hcclComm receive error,tag[%s], "\
        "outputPtr[%p], count[%llu], dataType[%s], srcRank[%u],", HCOM_ERROR_CODE(ret), tag,
        outputPtr, count, GetDataTypeEnumStr(dataType).c_str(), srcRank), ret);

    CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(tag));
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_RECEIVE, beginTime, count, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("HcclCommGraphReceive success,time[%lld]us, tag[%s], outputPtr[%p], count[%llu], dataType[%s], "\
        "srcRank[%u], srTag[%u], opBaseHcom[%lld],", DURATION_US(TIME_NOW() - startut), tag, outputPtr,
        count, GetDataTypeEnumStr(dataType).c_str(), srcRank, srTag, opBaseHcom);

    return HCCL_SUCCESS;
}
HcclResult HcclCommGraphGetAlltoAllStagedWorkSpaceMemSize(s64 opBaseHcom, u64 *sendCounts, u64 *sdispls,
    HcclDataType sendType, u64 *recvCounts, u64 *rdispls, HcclDataType recvType, u64 &memSize)
{
    CHK_PTR_NULL(sendCounts);
    CHK_PTR_NULL(sdispls);
    CHK_PTR_NULL(recvCounts);
    CHK_PTR_NULL(rdispls);
    CHK_RET(HcomCheckDataType(sendType));
    CHK_RET(HcomCheckDataType(recvType));

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    CHK_RET(hcclComm->GetAlltoAllStagedWorkSpaceMemSize(sendCounts, sdispls, sendType,
        recvCounts, rdispls, recvType, memSize));
    return HCCL_SUCCESS;
}
HcclResult HcclCommGraphSetWorkspaceResource(const std::string &tag, s64 opBaseHcom, std::vector<rtStream_t> stream,
    void *memPtr, u64 maxSize)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));

    CHK_RET(hcclComm->SetWorkspaceResource(tag, memPtr, maxSize, stream));
    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphGetRankSize(s64 opBaseHcom, u32 *rankSize)
{
    RPT_INPUT_ERR(rankSize == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphGetRankSize", "rankSize", "nullptr", "please check rankSize"}));
    CHK_PTR_NULL(rankSize);
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    if (isAutoTuneModeOpen) {
        *rankSize = 1;
        return HCCL_SUCCESS;
    }
    HCCL_INFO("HcclCommGraphGetRankSize:opBaseHcom[%lld]", opBaseHcom);

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    CHK_RET(hcclComm->GetRankSize(*rankSize));

    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphGetRankId(s64 opBaseHcom, u32 *rankId)
{
    RPT_INPUT_ERR(rankId == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcclCommGraphGetRankId", "rankId", "nullptr", "please check rankId"}));
    CHK_PTR_NULL(rankId);
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    if (isAutoTuneModeOpen) {
        *rankId = 0;
        return HCCL_SUCCESS;
    }
    HCCL_INFO("HcclCommGraphGetRankId:opBaseHcom[%lld]", opBaseHcom);

    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    CHK_RET(hcclComm->GetUserRank(*rankId));

    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphGetWorkspaceSubStreamNum(s64 opBaseHcom, u64 &streamNum, u64 dataSize,
    HcclCMDType opType)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    return hcclComm->GetWorkspaceSubStreamNum(streamNum, dataSize, opType);
}

HcclResult HcclCommGraphGetAllReduceScratchSize(s64 opBaseHcom, const u32 count, const HcclDataType dataType,
    u64 &outScratchSize)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    return hcclComm->GetAllReduceScratchSize(count, dataType, outScratchSize);
}

HcclResult HcclCommGraphGetIdentifier(s64 opBaseHcom, std::string &identifier)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    identifier = hcclComm->GetIdentifier();
    return HCCL_SUCCESS;
}

HcclResult GenerateCclOpTag(const std::string &opType, int64_t &hcomComm, std::string& group, std::string &sTag)
{
    HcomOpTagInfo &opTagInfo = HcomGetCtxOpTagInfo();

    // middle 在获取到hcomComm时，等于identifier；在获取到group时，等于group
    std::string middle;
    if (hcomComm == static_cast<int64_t>(CommNumHcom::COMM_VALUE_DEFAULT)) {
        middle = group;
    } else {
        CHK_RET(HcclCommGraphGetIdentifier(hcomComm, middle));
    }
    std::hash<std::string> hashString;
    size_t nameHash = hashString(middle);
    sTag = opType + "_" + std::to_string(nameHash);
    // 多张图node name重复导致tag相同，因此对tag添加索引 tag = op type + node name + identifier name + index

    auto iter = opTagInfo.opIndex.find(middle);
    if (iter == opTagInfo.opIndex.end()) {
        opTagInfo.opIndex.insert({ middle, 0 });
        iter = opTagInfo.opIndex.find(middle);
        CHK_PRT_RET((iter == opTagInfo.opIndex.end()),
            HCCL_ERROR("[Generate][OpTag]generate tag fail. get the op index failed. ret[%d]", HCCL_E_INTERNAL),
            HCCL_E_INTERNAL);
        }

    sTag = sTag + "_" + std::to_string(iter->second++);

    HCCL_INFO("generate ccl op tag success, tag[%s]", sTag.c_str());
    return HCCL_SUCCESS;
}

HcclResult HcomGetDevId(const char *group, s32 *devId)
{
    /* 获取rankId */
    std::shared_ptr<hccl::hcclComm> hcclComm;
    if (HcclGetCommHandle(group, hcclComm) == HCCL_SUCCESS) {
        CHK_RET(hcclComm->GetDeviceId(*devId));
    } else {
        if (group == nullptr) {
            group = HCCL_WORLD_GROUP;
        }
        u32 rankId = 0;
        CHK_RET(HcomGetRankId(group, &rankId));
        u32 worldRankId = 0;
        CHK_RET(HcomGetWorldRankFromGroupRank(group, rankId, &worldRankId));
        HcomInfo &hcomInfo = HcomGetCtxHomInfo();

        for (auto it : hcomInfo.rankTable.rankList) {
            if (worldRankId == it.rankId) {
                u32 deviceLogicId = 0;
                CHK_RET(hrtGetDeviceIndexByPhyId(static_cast<u32>(it.deviceInfo.devicePhyId), deviceLogicId));
                *devId = static_cast<s32>(deviceLogicId);
                return HCCL_SUCCESS;
            }
        }
        HCCL_WARNING("[Get][DevId]rankList has no item with rankId[%u]", worldRankId);
    }

    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphGetDevId(s64 opBaseHcom, s32 *devId)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(hcclComm->GetDeviceId(*devId));
    HCCL_INFO("HcclCommGraphGetDevId devID[%d]", *devId);
    return HCCL_SUCCESS;
}

HcclResult HcomGetLocalRankSize(const char *group, u32 *localRankSize)
{
    RPT_INPUT_ERR(localRankSize == nullptr, "EI0003",
        std::vector<std::string>({ "ccl_op", "parameter", "value", "tips" }),
        std::vector<std::string>({ "HcomGetLocalRankSize", "localRankSize", "nullptr", "please check localRankSize" }));
    CHK_PTR_NULL(localRankSize);
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    if (isAutoTuneModeOpen) {
        *localRankSize = 1;
        return HCCL_SUCCESS;
    }
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Get][LocalRankSize]hcomInfo.pComm is null, "\
        "please check if the initialize process is called."), HCCL_E_PTR);
    HcclResult ret = HcomCheckGroupName(group);
    RPT_INPUT_ERR(ret != HCCL_SUCCESS,
        "EI0003", std::vector<std::string>({ "ccl_op", "parameter", "value", "tips" }),
        std::vector<std::string>({ "HcomGetLocalRankSize",
        "group",
        { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
        "please check group name" }));
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Get][LocalRankSize]errNo[0x%016llx] get local ranksize " \
        "group name is invalid", HCOM_ERROR_CODE(ret)), ret);

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;

    u32 rankSize = 0;
    u32 serverNum = 0;
    CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::RANK_SIZE_IN_GROUP, 0, &rankSize));
    CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::SERVER_NUM_IN_GROUP, 0, &serverNum));

    CHK_PRT_RET(serverNum == 0, HCCL_ERROR("[Get][LocalRankSize]errNo[0x%016llx] server num is zero",
        HCOM_ERROR_CODE(HCCL_E_PARA)), HCCL_E_PARA);

    *localRankSize = rankSize / serverNum;
    HCCL_INFO("hcom get local rank size success, group[%s]", strGroup.c_str());

    return HCCL_SUCCESS;
}

HcclResult HcomGetRankId(const char *group, u32 *rankId)
{
    RPT_INPUT_ERR(rankId == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),\
        std::vector<std::string>({"HcomGetRankId", "rankId", "nullptr", "please check rankId"}));
    CHK_PTR_NULL(rankId);
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    if (isAutoTuneModeOpen) {
        *rankId = 0;
        return HCCL_SUCCESS;
    }
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Get][RankId]hcomInfo.pComm is null, "\
        "please check if the initialize process is called."), HCCL_E_PTR);
    HcclResult ret = HcomCheckGroupName(group);
    RPT_INPUT_ERR(ret != HCCL_SUCCESS,
        "EI0003", std::vector<std::string>({ "ccl_op", "parameter", "value", "tips" }),
        std::vector<std::string>({ "HcomGetRankId",
        "group",
        { group, strnlen(group, GROUP_NAME_MAX_LEN + 1) },
        "please check group name" }));
    CHK_PRT_RET(ret != HCCL_SUCCESS,
        HCCL_ERROR("[Get][RankId]errNo[0x%016llx] get_rank_id group name is invalid", HCOM_ERROR_CODE(ret)), ret);

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;

    CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::RANK_ID_IN_GROUP, 0, rankId));

    HCCL_INFO("hcom get rank id success, group[%s]", strGroup.c_str());

    return HCCL_SUCCESS;
}

HcclResult HcomGetLocalRankId(const char *group, u32 *localRankId)
{
    CHK_PTR_NULL(localRankId);
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    if (isAutoTuneModeOpen) {
        *localRankId = 0;
        return HCCL_SUCCESS;
    }
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Get][LocalRankId]hcomInfo.pComm is null, "\
        "please check if the initialize process is called."), HCCL_E_PTR);
    CHK_RET(HcomCheckGroupName(group));

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;

    u32 rankId = 0;
    u32 localRankSize = 0;
    CHK_RET(GetGroupRankInfo(strGroup.c_str(), RankInfoType::RANK_ID_IN_GROUP, 0, &rankId));
    CHK_RET(HcomGetLocalRankSize(strGroup.c_str(), &localRankSize));

    CHK_PRT_RET(localRankSize == 0, HCCL_ERROR("[Get][LocalRankId]errNo[0x%016llx] local rank size is zero",
        HCOM_ERROR_CODE(HCCL_E_PARA)), HCCL_E_PARA);

    *localRankId = rankId % localRankSize;
    HCCL_INFO("hcom get local rank id success, group[%s]", strGroup.c_str());

    return HCCL_SUCCESS;
}

HcclResult GetRankListHeterog(u32 rankNum, const u32 *rankIds, HcclGroupParams &params);
HcclResult HcomCreateGroupImplHeterog(const std::string &group, const std::vector<u32> &rankIds)
{
    HcclUs startut = TIME_NOW();
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    std::string rankId;
    for (u32 i = 0; i < rankIds.size(); i++) {
        if (i < rankIds.size() - 1) {
            rankId += to_string(rankIds[i]) + ',';
        } else if (i == rankIds.size() - 1) {
            rankId += to_string(rankIds[i]);
        }
    }
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcomCreateGroup:group[%s], rankNum[%zu], rankIds[%s]", group.c_str(), rankIds.size(),
        rankId.c_str());

    if (hcomInfo.params.commWorkMode == HCCL_MODE_SCHED_OS ||
        hcomInfo.params.commWorkMode == HCCL_MODE_AI_CPU ||
        hcomInfo.params.commWorkMode == HCCL_MODE_PS) {
        CHK_PRT_RET(hcomInfo.pCommBase == nullptr,
            HCCL_ERROR("[Create][Group]hcomInfo.pCommBase is null, please check if the initialize process is called."),
            HCCL_E_PTR);
    } else {
        CHK_PRT_RET(hcomInfo.pComm == nullptr,
            HCCL_ERROR("[Create][Group]hcomInfo.pComm is null, please check if the initialize process is called."),
            HCCL_E_PTR);
    }

    CHK_PRT_RET(hcomInfo.rankTable.rankList.empty(),
        HCCL_ERROR("[Create][Group]group[%s] rankList is empty", group.c_str()), HCCL_E_INTERNAL);

    /* 已经存在的group不允许再次创建 */
    std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
    if (hcomInfo.hcomGroupMap.find(group) != hcomInfo.hcomGroupMap.end()) {
        HCCL_ERROR("[Create][Group]errNo[0x%016llx] group[%s] is already exist", HCOM_ERROR_CODE(HCCL_E_PARA),
            group.c_str());
        return HCCL_E_PARA;
    }
    groupParaLock.unlock();

    HcclGroupParams groupParamsTem;
    CHK_RET(GetRankListHeterog(rankIds.size(), rankIds.data(), groupParamsTem));

    // 如果是groupRank = INVALID_VALUE_RANKID，即本rank不参与create group
    if (groupParamsTem.groupRank == INVALID_VALUE_RANKID) {
        HCCL_ERROR("[Create][Group]errNo[0x%016llx] confirm groupRank from worldRank[%u] error",
            HCOM_ERROR_CODE(HCCL_E_NOT_FOUND), hcomInfo.params.rank);
        return HCCL_E_NOT_FOUND;
    }

    /* 入参的正确性由HCCL确保 */
    if (hcomInfo.params.commWorkMode == HCCL_MODE_SCHED_OS ||
        hcomInfo.params.commWorkMode == HCCL_MODE_PS) {
        CHK_RET(hcomInfo.pCommBase->CreateGroup(group, groupParamsTem.groupRank, hcomInfo.params.rank,
            groupParamsTem.groupRanks, groupParamsTem.pSubCommBase));
        CHK_SMART_PTR_NULL(groupParamsTem.pSubCommBase);
    } else if (hcomInfo.params.commWorkMode == HCCL_MODE_AI_CPU) {
        CHK_RET(hcomInfo.pCommBase->CreateGroup(group, groupParamsTem.groupRank, hcomInfo.params.rank,
            groupParamsTem.groupRanks, groupParamsTem.pSubComm));
        CHK_SMART_PTR_NULL(groupParamsTem.pSubComm);
    } else {
        CHK_RET(hcomInfo.pComm->CreateGroup(
            group, groupParamsTem.groupRank, hcomInfo.params.rank, groupParamsTem.groupRanks, groupParamsTem.pSubComm));
        CHK_SMART_PTR_NULL(groupParamsTem.pSubComm);
    }

    groupParaLock.lock();
    hcomInfo.hcomGroupMap.insert(std::make_pair(group, groupParamsTem));
    groupParaLock.unlock();

    HCCL_RUN_INFO("hcom create group[%s] success, take time [%lld]us",
        group.c_str(), DURATION_US(TIME_NOW() - startut));
    return HCCL_SUCCESS;
}

HcclResult HcomAbortGroup(const char *group)
{
    /* 调优模式直接返回success */
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    if (isAutoTuneModeOpen) {
        return HCCL_SUCCESS;
    }
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();

    if (hcomInfo.params.commWorkMode == HCCL_MODE_SCHED_OS ||
        hcomInfo.params.commWorkMode == HCCL_MODE_AI_CPU ||
        hcomInfo.params.commWorkMode == HCCL_MODE_PS) {
        CHK_PRT_RET(hcomInfo.pCommBase == nullptr, HCCL_ERROR("[Destroy][Group]hcomInfo.pCommBase is null, "\
        "please check if the initialize process is called."), HCCL_E_PTR);
    } else {
        CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Destroy][Group]hcomInfo.pComm is null, "\
        "please check if the initialize process is called."), HCCL_E_PTR);
    }

    HCCL_RUN_INFO("Entry-HcomAbortGroup : group[%s]", group);
    CHK_RET(DestroyFlag(group, true));
    u32 ref = 0;
    CHK_RET(HcomQueryGroupRef(group, ref));
    while (ref != 0) {
        if (hcomInfo.params.commWorkMode == HCCL_MODE_SCHED_OS ||
            hcomInfo.params.commWorkMode == HCCL_MODE_PS) {
            std::shared_ptr<hccl::HcclCommBase> hcclComm = nullptr;
            CHK_RET(HcpuGetCommByGroup(group, hcclComm));
            CHK_RET(hcclComm->Break());
            SaluSleep(ONE_HUNDRED_MICROSECOND_OF_USLEEP);
            CHK_RET(HcomQueryGroupRef(group, ref));
        } else {
            std::shared_ptr<hccl::hcclComm> hcclComm = nullptr;
            CHK_RET(HcomGetCommByGroup(group, hcclComm));
            SaluSleep(ONE_HUNDRED_MICROSECOND_OF_USLEEP);
            CHK_RET(HcomQueryGroupRef(group, ref));
        }
    }

    HCCL_RUN_INFO("hcom abort group[%s] success.", group);
    return HCCL_SUCCESS;
}

HcclResult HcomDestroyGroupImplHeterog(const std::string &group)
{
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcomDestroyGroup:group[%s]", group.c_str());
    CHK_RET(HcomAbortGroup(group.c_str()));

    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
    auto iter = hcomInfo.hcomGroupMap.find(group);
    if (iter == hcomInfo.hcomGroupMap.end()) {
        HCCL_ERROR("[Destroy][Group]errNo[0x%016llx] group[%s] is not exist", HCOM_ERROR_CODE(HCCL_E_PARA),
            group.c_str());
        return HCCL_E_PARA;
    }

    if (hcomInfo.params.commWorkMode == HCCL_MODE_SCHED_OS ||
        hcomInfo.params.commWorkMode == HCCL_MODE_AI_CPU ||
        hcomInfo.params.commWorkMode == HCCL_MODE_PS) {
        CHK_RET(hcomInfo.pCommBase->DestroyGroup(group));
    } else {
        CHK_RET(hcomInfo.pComm->DestroyGroup(group));
    }

    (iter->second).groupRanks.clear();  // 清除该服务器内相关group的对应信息

    hcomInfo.hcomGroupMap.erase(group);
    groupParaLock.unlock();

    HCCL_RUN_INFO("hcom destroy group[%s] success.", group.c_str());
    return HCCL_SUCCESS;
}

bool HcomCallBackGroupIsInitHeterog(HcomInfo &hcomInfo)
{
    if (hcomInfo.pCommBase == nullptr) {
        return false;
    } else {
        return true;
    }
}

HcclResult GetRankListHeterog(u32 rankNum, const u32 *rankIds, HcclGroupParams &params)
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    std::vector<RankInfo_t> rankList;
    params.totalRanks = rankNum;
    params.worldRank = hcomInfo.params.rank;
    params.groupRank = INVALID_VALUE_RANKID;
    for (u32 i = 0; i < rankNum; i++) {
        params.groupRanks.push_back(rankIds[i]);
    }

    std::sort(params.groupRanks.begin(), params.groupRanks.end());
    if (params.groupRanks[rankNum - 1] >= hcomInfo.rankTable.rankNum) {
        HCCL_ERROR("[get][RankList]errNo[0x%016llx] groupRanks[%u]:%u is invalid", HCOM_ERROR_CODE(HCCL_E_PARA),
            rankNum - 1, params.groupRanks[rankNum - 1]);
        return HCCL_E_PARA;
    }
    if (hcomInfo.rankTable.rankList.size() <= params.groupRanks[0]) {
        HCCL_ERROR("[get][RankList]errNo[0x%016llx] groupRanks[0] is invalid:[%u]", HCOM_ERROR_CODE(HCCL_E_PARA),
            params.groupRanks[0]);
        return HCCL_E_PARA;
    }
    // groupRanks 个数已经校验非0
    std::string serverId = hcomInfo.rankTable.rankList[params.groupRanks[0]].serverId;
    u32 serverNum = 1;  // severNum初始值应为1，代表groupId为0的serverId;
    RankInfo_t rankInfo;
    for (u32 i = 0; i < rankNum; i++) {
        rankInfo = hcomInfo.rankTable.rankList[params.groupRanks[i]];
        // 校验worldRankID
        if (rankInfo.rankId != params.groupRanks[i]) {
            HCCL_ERROR("[get][RankList]errNo[0x%016llx] in rankList, worldRanks[%u] is invalid",
                HCOM_ERROR_CODE(HCCL_E_PARA), rankInfo.rankId);
            return HCCL_E_PARA;
        }
        if (params.groupRanks[i] == params.worldRank) {
            params.groupRank = i;
        }
        if (rankInfo.serverId != serverId) {
            serverNum++;
            serverId = rankInfo.serverId;
        }
        rankInfo.rankId = i;           // 放入groupRankid
        rankList.push_back(rankInfo);  // ranktable中的ranklist是以rankid的顺序排列的
        rankInfo.serverId = "";        // 释放前先指空字符串
    }
    params.serverNum = serverNum;
    bool isStandardCard = false;
    if (hcomInfo.params.commWorkMode == HCCL_MODE_SCHED_OS ||
        hcomInfo.params.commWorkMode == HCCL_MODE_AI_CPU ||
        hcomInfo.params.commWorkMode == HCCL_MODE_PS) {
        CHK_RET((hcomInfo.pCommBase)->IsStandardCard(isStandardCard));
    } else {
        CHK_RET(hcomInfo.pComm->IsStandardCard(isStandardCard));
    }
    if (!isStandardCard && hcomInfo.params.deviceType != DevType::DEV_TYPE_910B &&
        hcomInfo.params.deviceType != DevType::DEV_TYPE_910_93) {
        CHK_RET(CheckRankTableConfigInfo(rankList, rankNum, serverNum));
    }
    return HCCL_SUCCESS;
}

HcclResult HcomGetSplitStrategy(const char *group, const struct model_feature *feature,
    std::vector<u32>& segmentIdx, bool &configured, GradSplitForceMode force, OriginalGraphShapeType shapeType)
{
    CHK_PTR_NULL(feature);
    CHK_PTR_NULL(feature->model_name);
    CHK_PTR_NULL(feature->gradient_size);
    CHK_PTR_NULL(feature->gradient_time);

    bool bRet = feature->gradient_num == 0;
    CHK_PRT_RET(
        bRet, HCCL_ERROR("[Get][SplitStrategy]errNo[0x%016llx] gradient num is zero", HCOM_ERROR_CODE(HCCL_E_PARA)),
        HCCL_E_PARA);
    CHK_RET(HcomCheckGroupName(group));

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    /* 接口交互信息日志 */
    HCCL_RUN_INFO("Entry-HcomGetSplitStrategy:group[%s], feature[%p]", strGroup.c_str(), feature);

    /* 获取梯度切分策略 */
    CHK_RET(GetGradientSegment(strGroup, feature, segmentIdx, configured, force, shapeType));

    return HCCL_SUCCESS;
}

HcclResult GetGradientSegment(const std::string &group, const struct model_feature *feature,
    std::vector<u32>& segmentList, bool &configured, GradSplitForceMode force, OriginalGraphShapeType shapeType)
{
    HcclResult ret;
    HCCL_INFO("<gradient_segment group %s, model gradient num %u, model name %s>", group.c_str(), feature->gradient_num,
        feature->model_name);
    /* 分段算法实现 */
    std::unique_ptr<hccl::GradientSegment> segmentImpl;
    segmentImpl.reset(new (std::nothrow) GradientSegment());
    CHK_SMART_PTR_NULL(segmentImpl);

    /* 校验基于总层数索引是否正确 */
    std::unique_lock<std::mutex> segmentIdxMapLock(g_segmentIdxMapLock);
    auto gIdxSearch = g_segmentIdxMap.find(group);
    if (gIdxSearch != g_segmentIdxMap.end()) {
        bool bRet = (gIdxSearch->second.size() != 0) && (gIdxSearch->second.back() != (feature->gradient_num - 1));
        CHK_PRT_RET(bRet, HCCL_ERROR("[Get][GradientSegment]illegal segmentIndex maxVal=%u should be equal %u",
            gIdxSearch->second.back(), feature->gradient_num - 1), HCCL_E_PARA);
    }
    segmentIdxMapLock.unlock();
    ret = segmentImpl->GetGradientSegmentExecutor(group, feature, segmentList, configured, force,
        shapeType);
    if (ret == HCCL_SUCCESS) {
        std::string printStr;
        u32 baseIndex = 0;
        for (u32 i = 0; i < segmentList.size(); i++) {
            printStr.append("[");
            printStr.append(std::to_string(baseIndex));
            printStr.append(",");
            printStr.append(std::to_string(segmentList[i]));
            printStr.append("] ");
            baseIndex = segmentList[i] + 1;
        }
        HCCL_RUN_INFO("gradient segment result: segment num: %zu, segment index list: %s ",    \
            segmentList.size(), printStr.c_str());
    }
    return ret;
}

// 取得所需的从stream数目
HcclResult HcomGetWorkspaceSubStreamNum(const char *group, u64 &streamNum, u64 dataSize, HcclCMDType optype)
{
    std::shared_ptr<hccl::hcclComm> hcclComm{};
    std::shared_ptr<hccl::HcclCommBase> hcclCommbase{};
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    hcclComm = hcomInfo.pComm;
    hcclCommbase = hcomInfo.pCommBase;
    CHK_RET(HcomCheckGroupName(group));
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    if (strGroup != HCCL_WORLD_GROUP &&
            (hcomInfo.pCommBase != nullptr || hcomInfo.pComm != nullptr)) {
        std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
        auto iter = hcomInfo.hcomGroupMap.find(strGroup);
        if (iter != hcomInfo.hcomGroupMap.end()) {
            hcclComm = (iter->second).pSubComm;
            hcclCommbase = (iter->second).pSubCommBase;
        } else {
            HCCL_WARNING("[GetWorkspaceSubStreamNum], please check if the initialize process is called.");
            streamNum = 0;
        }
    } else if (hcomInfo.pComm == nullptr && hcomInfo.pCommBase == nullptr) {
        HcclResult ret = HcclGetCommHandle(group, hcclComm);
        CHK_PRT_RET(ret != HCCL_SUCCESS,
            HCCL_WARNING("[GetWorkspaceSubStreamNum], please check if the initialize process is called."),
            HCCL_SUCCESS);
    }
    CHK_PRT_RET(hcclComm == nullptr && hcclCommbase == nullptr,
        HCCL_ERROR("[GetWorkspaceSubStreamNum] Get Comm is null"), HCCL_E_PTR);
    if (hcclComm != nullptr) {
        return hcclComm->GetWorkspaceSubStreamNum(streamNum, dataSize, optype);
    } else {
        return hcclCommbase->GetWorkspaceSubStreamNum(streamNum, dataSize, optype);
    }
}

HcclResult HcomGetWorkspaceMemSize(const std::string &opType, u64 count, HcclDataType dataType, const char *group,
    u64 &memSize)
{
    u32 rankSize = 0;
    std::shared_ptr<hccl::hcclComm> hcclComm{};
    std::shared_ptr<hccl::HcclCommBase> hcclCommbase{};
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    if (hcomInfo.pComm != nullptr) {
        hcclComm = hcomInfo.pComm;
        if (group == nullptr) {
            group = HCCL_WORLD_GROUP;
        }
        CHK_RET(HcomGetRankSize(group, &rankSize));
        CHK_RET(hcclComm->GetWorkspaceMemSize(opType, count, dataType, rankSize, memSize));
    } else if (hcomInfo.pCommBase != nullptr) {
        hcclCommbase = hcomInfo.pCommBase;
        if (group == nullptr) {
            group = HCCL_WORLD_GROUP;
        }
        CHK_RET(HcomGetRankSize(group, &rankSize));
        CHK_RET(hcclCommbase->GetWorkspaceMemSize(opType, count, dataType, rankSize, memSize));
    } else if (HcclGetCommHandle(group, hcclComm) == HCCL_SUCCESS) {
        CHK_RET(hcclComm->GetRankSize(rankSize));
        CHK_RET(hcclComm->GetWorkspaceMemSize(opType, count, dataType, rankSize, memSize));
    } else {
        HCCL_WARNING("[GetWorkspaceMemSize] please check if the initialize process is called.");
        memSize = 0;
        return HCCL_SUCCESS;
    }
    /* 用户申请内存,获取memSize大小 */

    CHK_PRT_RET(memSize > DEVICE_MEMORY_MAX_ALLOC_SIZE,
        HCCL_ERROR("[GetWorkspaceMemSize]workspace memory size is over than %llu bytes.", DEVICE_MEMORY_MAX_ALLOC_SIZE),
        HCCL_E_PARA);
    return HCCL_SUCCESS;
}

HcclResult HcomGetAlltoAllStagedWorkSpaceMemSize(const char *group, u64 *sendCounts, u64 *sdispls,
    HcclDataType sendType, u64 *recvCounts, u64 *rdispls, HcclDataType recvType, u64 &memSize)
{
    CHK_PTR_NULL(sendCounts);
    CHK_PTR_NULL(sdispls);
    CHK_PTR_NULL(recvCounts);
    CHK_PTR_NULL(rdispls);
    CHK_RET(HcomCheckDataType(sendType));
    CHK_RET(HcomCheckDataType(recvType));

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    CHK_RET(hcclComm->GetAlltoAllStagedWorkSpaceMemSize(sendCounts, sdispls, sendType,
        recvCounts, rdispls, recvType, memSize));
    return HCCL_SUCCESS;
}

HcclResult HcomGetAlltoAllvcStagedWorkSpaceMemSize(const char *group,
    std::vector<SendRecvInfo> &allMeshAggregationSendRecvInfo, u64 &memSize)
{
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    CHK_RET(hcclComm->GetAlltoAllStagedWorkSpaceMemSize(allMeshAggregationSendRecvInfo, memSize));
    return HCCL_SUCCESS;
}

HcclResult HcomGetAllReduceScratchSize(const char *group, const u32 count, const HcclDataType dataType,
    u64 &outScratchSize)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    if (hcomInfo.pComm != nullptr) {
        hcclComm = hcomInfo.pComm;
        CHK_RET(HcomCheckGroupName(group));
        std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
        if (strGroup == HCCL_WORLD_GROUP) {
            hcclComm = hcomInfo.pComm;
        } else {
            std::unique_lock<std::mutex> groupParaLock(hcomInfo.groupParamsLock);
            auto iter = hcomInfo.hcomGroupMap.find(strGroup);
            if (iter != hcomInfo.hcomGroupMap.end()) {
                hcclComm = (iter->second).pSubComm;
                CHK_PRT_RET(hcclComm == nullptr, HCCL_ERROR("[Get][CommByGroup] Get Comm is null"), HCCL_E_PTR);
            } else {
                u64 memSize = SIZE_TABLE[dataType] * count;
                const u32 DEVICE_EIGHT = 8;
                if (memSize <= HCCL_SMALL_COUNT_GRAPH_64_KB) {
                    // 小数据
                    outScratchSize = memSize * (DEVICE_EIGHT - 1);
                }
                HCCL_DEBUG("[HcomGetAllReduceScratchSize] outScratchSize %llu", outScratchSize);
                groupParaLock.unlock();
                return HCCL_SUCCESS;
            }
            groupParaLock.unlock();
        }
    } else if (HcclGetCommHandle(group, hcclComm) != HCCL_SUCCESS) {
        HCCL_WARNING("[GetAllReduceScratchSize], please check if the initialize process is called.");
        outScratchSize = 0;
        return HCCL_SUCCESS;
    }
    return hcclComm->GetAllReduceScratchSize(count, dataType, outScratchSize);
}


HcclResult HcomGetCCLBufferAvailableSize(u64 &size)
{
    size = GetExternalInputCCLBuffSize() - CCL_COMM_INBUFFER_UNALIGNED_RESERVE_SIZE;
    return HCCL_SUCCESS;
}

HcclResult HcomCheckCommValidity(const char* group)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    if (HcomGetCommByGroup(group, hcclComm) != HCCL_SUCCESS) {
        std::shared_ptr<hccl::HcclCommBase> hcclCommcpu;
        if (HcpuGetCommByGroup(group, hcclCommcpu) != HCCL_SUCCESS) {
            return HCCL_E_PTR;
        }
    }

    return HCCL_SUCCESS;
}

HcclResult HcomSetWorkspaceResource(const std::string &tag, const char *group, std::vector<rtStream_t> stream,
    void *memPtr, u64 maxSize)
{
    if (group == nullptr) {
        group = HCCL_WORLD_GROUP;
    }

    std::shared_ptr<hccl::hcclComm> hcclComm;
    if (HcomGetCommByGroup(group, hcclComm) == HCCL_SUCCESS) {
        /* 设定 workspace 内存资源 */
        CHK_RET(hcclComm->SetWorkspaceResource(tag, memPtr, maxSize, stream));
    } else {
        std::shared_ptr<hccl::HcclCommBase> hcclCommcpu;
        CHK_RET(HcpuGetCommByGroup(group, hcclCommcpu));
        CHK_RET(hcclCommcpu->SetWorkspaceResource(tag, memPtr, maxSize, stream));
    }
    return HCCL_SUCCESS;
}

HcclResult HcomSetAttachedStream(const char *group, const std::vector<rtStream_t> &stream)
{
    if (group == nullptr) {
        group = HCCL_WORLD_GROUP;
    }

    std::shared_ptr<hccl::hcclComm> hcclComm = nullptr;
    if (HcomGetCommByGroup(group, hcclComm) == HCCL_SUCCESS) {
        CHK_RET(hcclComm->SetAttachedStream(stream));
    } else {
        // HcclCommBase 场景暂是不支持设置附属从流
        HCCL_WARNING("[HcomSetAttachedStream] HcclCommBase now don't support set attached stream");
        return HCCL_SUCCESS;
    }

    return HCCL_SUCCESS;
}

HcclResult HcclCommSetAttachedStream(s64 opBaseHcom, const std::vector<rtStream_t> &stream)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_PTR_NULL(hcclComm);
    CHK_RET(hcclComm->SetAttachedStream(stream));

    return HCCL_SUCCESS;
}

HcclResult HcomGetRemoteAccess(std::shared_ptr<hccl::RemoteAccess> &remoteAccess)
{
    shared_ptr<RemoteAccess> ctxRemoteAccess = HcomGetCtxRemoteAccess();
    if (ctxRemoteAccess == nullptr) {
        HCCL_ERROR("[Get][RemoteAccess]Unable to get remote access impl, the impl is null!");
        return HCCL_E_UNAVAIL;
    }
    remoteAccess = ctxRemoteAccess;
    return HCCL_SUCCESS;
}

DevType HcomGetDeviceType()
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    return hcomInfo.params.deviceType;
}

void HcomSetAutoTuneMode(bool autoTuneMode)
{
    bool &isAutoTuneModeOpen = HcomGetCtxAutoTuneMode();
    isAutoTuneModeOpen = autoTuneMode;
}

HcclResult HcomSetExecTimeOut(const char *execTimeOut)
{
    CHK_RET(SetHccLExecTimeOut(execTimeOut, HcclExecTimeoutSet::HCCL_EXEC_TIMEOUT_SET_BY_OPTIONS));
    return HCCL_SUCCESS;
}

HcclResult HcomSetAlgorithm(const char* algo)
{
    CHK_PTR_NULL(algo);

    HCCL_RUN_INFO("Entry-HcomSetAlgorithm:algo[%s]", algo);
    CHK_RET(SetHcclAlgoConfig(algo));
    return HCCL_SUCCESS;
}

HcclResult HcomSetPsTableNum(u32 tableNum)
{
    HCCL_RUN_INFO("HcomSetPsTableNum get tableNum[%u]", tableNum);
    SetEsMaxPsTable(tableNum);
    return HCCL_SUCCESS;
}

HcclResult HcomSetDeterministic(bool isDeterministic)
{
    CHK_RET(SetDeterministic(isDeterministic));
    return HCCL_SUCCESS;
}

HcclResult HcomGetAlgorithm(u32 level, std::string &algo)
{
    algo = "none";
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    CHK_PRT_RET(hcomInfo.pComm == nullptr,
        HCCL_ERROR("[Get][Algorithm]hcomInfo.pComm is null, "\
        "please check if the initialize process is called."),
        HCCL_E_PTR);
    AlgType algType;
    CHK_RET(hcomInfo.pComm->GetAlgType(algType, HcclCMDType::HCCL_CMD_ALL));
    if (level == 0) {
        if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_WHOLE_RING ||
            algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_8P_RING ||
            algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_4P_RING ||
            algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_NP_SINGLE_RING ||
            algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_NP_DOUBLE_RING) {
            algo = "ring";
        } else if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_NP_MESH ||
            algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_4P_MESH ||
            algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_2P_MESH ||
            algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_1P_MESH) {
            algo = "mesh";
        } else if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_RESERVED &&
            algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
            algo = "NHR";
        } else if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_RESERVED &&
            algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
            algo = "NHR_V1";
        } else if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_RESERVED &&
            algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC) {
            algo = "AHC";
        } else if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_RESERVED &&
            algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE) {
            algo = "AHC_BROKE";
        } else if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_RESERVED &&
            algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
            algo = "NB";
        } else {
            HCCL_WARNING("[Get][Algorithm] No valid Level 0 AlgType, which is [%d]",
                static_cast<s32>(algType.algoLevel0));
            return HCCL_E_NOT_FOUND;
        }
    } else if (level == 1) {
        if (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_WHOLE_RING ||
            (algType.algoLevel0 == AlgTypeLevel0::ALG_LEVEL0_RESERVED && 
            algType.algoLevel1 != AlgTypeLevel1::ALG_LEVEL1_RESERVED)) {
            algo = "none";
        } else if (algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_HD) {
            algo = "H-D";
        } else if (algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_PIPELINE ||
            algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_RING) {
            algo = "ring";
        } else if (algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR) {
            algo = "NHR";
        } else if (algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NHR_V1) {
            algo = "NHR_V1";
        } else if (algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC) {
            algo = "AHC";
        } else if (algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_AHC_BROKE) {
            algo = "AHC_BROKE";
        } else if (algType.algoLevel1 == AlgTypeLevel1::ALG_LEVEL1_NB) {
            algo = "NB";
        } else {
            HCCL_WARNING("[Get][Algorithm] No valid Level 1 AlgType, which is [%d]",
                static_cast<s32>(algType.algoLevel1));
            return HCCL_E_NOT_FOUND;
        }
    }
    return HCCL_SUCCESS;
}

HcclResult HcomGetBandWidthPerNPU(u32 level, float &bandWidth)
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    CHK_PRT_RET(hcomInfo.pComm == nullptr, HCCL_ERROR("[Get][BandWidth]hcomInfo.pComm is null, "\
        "please check if the initialize process is called."), HCCL_E_PTR);
    CHK_RET(hcomInfo.pComm->GetBandWidthPerNPU(level, bandWidth));
    return HCCL_SUCCESS;
}

HcclResult HcomReleaseSubComms()
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    if (hcomInfo.pComm) {
        CHK_RET(hcomInfo.pComm->ReleaseSubComms());
    }

    auto iter = hcomInfo.hcomGroupMap.begin();
    while (iter != hcomInfo.hcomGroupMap.end()) {
        if (iter->second.pSubComm) {
            CHK_RET(iter->second.pSubComm->ReleaseSubComms());
        }
        iter++;
    }
    return HCCL_SUCCESS;
}

HcclResult HcomAlltoAllV(const void *sendBuf, const void *sendCounts, const void *sdispls, HcclDataType sendType,
                         const void *recvBuf, const void *recvCounts, const void *rdispls, HcclDataType recvType,
                         const char *group, rtStream_t stream, const char *tag)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    // 入参合法性校验
    CHK_PTR_NULL(sendCounts);
    CHK_PTR_NULL(sdispls);
    CHK_PTR_NULL(recvCounts);
    CHK_PTR_NULL(rdispls);
    CHK_RET(HcomCheckOpParam(tag, 0, sendType, group, stream));
    CHK_RET(HcomCheckDataType(recvType));
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    // 根据ranksize校验相关入参
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    u32 rankSize = 0;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = 0;
    CHK_RET(hcclComm->GetUserRank(rankId));
    CHK_RET(HcomCheckAlltoAllVExternalMem(sendBuf, sendCounts, recvBuf, recvCounts, rankSize));
    if (sendBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(sendBuf));
    }
    if (recvBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(recvBuf));
    }

    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAlltoAllV:tag[%s], sendBuf[%p], recvBuf[%p], sendCounts[%p], recvCounts[%p],"
        "sendType[%s], recvType[%s], group[%s], streamId[%d], deviceLogicId[%d]",
        tag, sendBuf, recvBuf, sendCounts, recvCounts, GetDataTypeEnumStr(sendType).c_str(),
        GetDataTypeEnumStr(recvType).c_str(), strGroup.c_str(), streamId, deviceLogicId);

    /* 记录cclBufferSize用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_ALLTOALLV,
        tag, 0, HCCL_DATA_TYPE_RESERVED, hcclComm->GetConfigInCCLbufferSize(), 0,
        strGroup.c_str(), hcclComm->GetRankTableCrc()));
    HcclWorkflowMode mode = GetWorkflowMode();
    CHK_PRT_RET(mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_RESERVED, HCCL_ERROR("Invalid Workflow Mode[%d]", mode),
        HCCL_E_INTERNAL);
    HCCL_PROFILER_ADD_TAG(tag, strGroup, mode);
    HCCL_PROFILER_ADD_TAG(HCCL_ALLTOALL_PARA_ALLGATHER, strGroup, mode);
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());

    /* 入参的正确性由HCCL确保 */
    HCCL_PROFILER_ADD_OPDATA_OP(tag, 0, sendBuf, recvBuf, sendType, INVALID_VALUE_RANKID, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, rankId);

    CHK_RET(hcclComm->AlltoAllV(sendBuf, sendCounts, sdispls, sendType, recvBuf, recvCounts, rdispls, recvType,
                                stream, tag));

    u64 sendCount = 0;
    for (u32 i = 0; i < rankSize; i++) {
        sendCount += *(static_cast<const u64 *>(sendCounts) + i);
    }
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(HCCL_ALLTOALL_PARA_ALLGATHER);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_ALLTOALLV, beginTime, sendCount, sendType));
    /* 关键状态记录 */
    HcclUs endut = TIME_NOW();
    HCCL_RUN_INFO("HcomAlltoAllV success,take time [%lld]us, tag[%s], sendBuf[%p], recvBuf[%p], sendCounts[%p], "\
               "recvCounts[%p], sendType[%s], recvType[%s], group[%s], streamId[%d]", DURATION_US(endut - startut),
               tag, sendBuf, recvBuf, sendCounts, recvCounts, GetDataTypeEnumStr(sendType).c_str(),
               GetDataTypeEnumStr(recvType).c_str(), strGroup.c_str(), streamId);
    return HCCL_SUCCESS;
}

HcclResult HcomAlltoAllVC(const void *sendBuf, const void *sendCountMatrix, HcclDataType sendType,
    const void *recvBuf, HcclDataType recvType, const char *group, rtStream_t stream, const char *tag)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    // 入参合法性校验
    CHK_PTR_NULL(sendCountMatrix);
    CHK_RET(HcomCheckOpParam(tag, 0, sendType, group, stream));
    CHK_RET(HcomCheckDataType(recvType));
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    // 根据ranksize校验相关入参
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;

    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    u32 rankSize = 0;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 userRank = 0;
    hcclComm->GetGroupRank(userRank);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    u32 rank = hcomInfo.params.userRank;
    CHK_RET(HcomCheckAlltoAllVCExternalMem(sendBuf, sendCountMatrix, recvBuf, rankSize, rank));
    if (sendBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(sendBuf));
    }
    if (recvBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(recvBuf));
    }

    u64 sendCountMatrixHash;
    HcomGetHashFromSendCountMatrix(sendCountMatrixHash, sendCountMatrix, rankSize, tag);

    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAlltoAllVC:tag[%s], sendBuf[%p], sendCountMatrixHash[%llu], sendType[%s], "\
               "recvBuf[%p], recvType[%s], group[%s], streamId[%d], deviceLogicId[%d]",
               tag, sendBuf, sendCountMatrixHash, GetDataTypeEnumStr(sendType).c_str(),
               recvBuf, GetDataTypeEnumStr(recvType).c_str(), strGroup.c_str(), streamId, deviceLogicId);

    /* 记录cclBufferSize用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_ALLTOALLVC,
        tag, 0, HCCL_DATA_TYPE_RESERVED, hcclComm->GetConfigInCCLbufferSize(), 0, nullptr,
        hcclComm->GetRankTableCrc()));
    HcclWorkflowMode mode = GetWorkflowMode();
    CHK_PRT_RET(mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_RESERVED, HCCL_ERROR("Invalid Workflow Mode[%d]", mode),
        HCCL_E_INTERNAL);
    HCCL_PROFILER_ADD_TAG(tag, strGroup, mode);
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());

    HCCL_PROFILER_ADD_OPDATA_OP(tag, 0, sendBuf, recvBuf, sendType, INVALID_VALUE_RANKID, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, userRank);

    /* 入参的正确性由HCCL确保 */
    CHK_RET(hcclComm->AlltoAllVC(sendBuf, sendCountMatrix, sendType, recvBuf, recvType, stream, tag));

    u64 sendCount = 0;
    for (u32 i = 0; i < rankSize; i++) {
        sendCount += *(static_cast<const u64 *>(sendCountMatrix) + userRank * rankSize + i);
    }
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_ALLTOALLVC, beginTime, sendCount, sendType));
    /* 关键状态记录 */
    HcclUs endut = TIME_NOW();
    HCCL_RUN_INFO("HcomAlltoAllVC success,take time [%lld]us, tag[%s], sendBuf[%p], sendCountMatrix[%p], "\
        "sendType[%s], recvBuf[%p], recvType[%s], group[%s], streamId[%d]", DURATION_US(endut - startut),
        tag, sendBuf, sendCountMatrix, GetDataTypeEnumStr(sendType).c_str(), recvBuf,
        GetDataTypeEnumStr(recvType).c_str(), strGroup.c_str(), streamId);
    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphAlltoAllV(const void *sendBuf, const void *sendCounts, const void *sdispls,
    HcclDataType sendType, const void *recvBuf, const void *recvCounts, const void *rdispls,
    HcclDataType recvType, s64 opBaseHcom, rtStream_t stream, const char *tag)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    // 入参合法性校验
    CHK_PTR_NULL(sendCounts);
    CHK_PTR_NULL(sdispls);
    CHK_PTR_NULL(recvCounts);
    CHK_PTR_NULL(rdispls);

    CHK_RET(HcomCheckOpParam(tag, 0, sendType, stream));
    CHK_RET(HcomCheckDataType(recvType));
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    // 根据ranksize校验相关入参
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    u32 rankSize = 0;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 rankId = 0;
    CHK_RET(hcclComm->GetUserRank(rankId));
    CHK_RET(HcomCheckAlltoAllVExternalMem(sendBuf, sendCounts, recvBuf, recvCounts, rankSize));
    if (sendBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(sendBuf));
    }
    if (recvBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(recvBuf));
    }

    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAlltoAllV:tag[%s], sendBuf[%p], recvBuf[%p], sendCounts[%p], recvCounts[%p],"
        "sendType[%s], recvType[%s], streamId[%d], deviceLogicId[%d]",
        tag, sendBuf, recvBuf, sendCounts, recvCounts, GetDataTypeEnumStr(sendType).c_str(),
        GetDataTypeEnumStr(recvType).c_str(), streamId, deviceLogicId);

    /* 记录cclBufferSize用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_ALLTOALLV,
        tag, 0, HCCL_DATA_TYPE_RESERVED, hcclComm->GetConfigInCCLbufferSize(), 0, nullptr,
        hcclComm->GetRankTableCrc()));
    HcclWorkflowMode mode = GetWorkflowMode();
    CHK_PRT_RET(mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_RESERVED, HCCL_ERROR("Invalid Workflow Mode[%d]", mode),
        HCCL_E_INTERNAL);
    HCCL_PROFILER_ADD_TAG(tag, hcclComm->GetIdentifier(), mode);
    HCCL_PROFILER_ADD_TAG(HCCL_ALLTOALL_PARA_ALLGATHER, hcclComm->GetIdentifier(), mode);
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());

    HCCL_PROFILER_ADD_OPDATA_OP(tag, 0, sendBuf, recvBuf, sendType, INVALID_VALUE_RANKID, hcclComm->GetIdentifier(), \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(hcclComm->GetIdentifier(), rankSize, rankId);

    /* 入参的正确性由HCCL确保 */
    CHK_RET(hcclComm->AlltoAllV(sendBuf, sendCounts, sdispls, sendType, recvBuf, recvCounts, rdispls, recvType,
                                stream, tag));
    u64 sendCount = 0;
    for (u32 i = 0; i < rankSize; i++) {
        sendCount += *(static_cast<const u64 *>(sendCounts) + i);
    }
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(HCCL_ALLTOALL_PARA_ALLGATHER);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_ALLTOALLV, beginTime, sendCount, sendType));
    /* 关键状态记录 */
    HcclUs endut = TIME_NOW();
    HCCL_RUN_INFO("HcomAlltoAllV success,take time [%lld]us, tag[%s], sendBuf[%p], recvBuf[%p], sendCounts[%p], "
        "recvCounts[%p], sendType[%s], recvType[%s], streamId[%d]",
        DURATION_US(endut - startut), tag, sendBuf, recvBuf, sendCounts, recvCounts,
        GetDataTypeEnumStr(sendType).c_str(), GetDataTypeEnumStr(recvType).c_str(), streamId);
    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphAlltoAllVC(const void *sendBuf, const void *sendCountMatrix, HcclDataType sendType,
    const void *recvBuf, HcclDataType recvType, s64 opBaseHcom, rtStream_t stream, const char *tag)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    // 入参合法性校验
    CHK_PTR_NULL(sendCountMatrix);

    CHK_RET(HcomCheckOpParam(tag, 0, sendType, stream));
    CHK_RET(HcomCheckDataType(recvType));
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    // 根据ranksize校验相关入参
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    u32 rankSize = 0;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    u32 userRank = 0;
    hcclComm->GetGroupRank(userRank);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    u32 rank = hcomInfo.params.userRank;
    CHK_RET(HcomCheckAlltoAllVCExternalMem(sendBuf, sendCountMatrix, recvBuf, rankSize, rank));
    if (sendBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(sendBuf));
    }
    if (recvBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(recvBuf));
    }

    u64 sendCountMatrixHash;
    HcomGetHashFromSendCountMatrix(sendCountMatrixHash, sendCountMatrix, rankSize, tag);

    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAlltoAllVC:tag[%s], sendBuf[%p], sendCountMatrixHash[%llu], sendType[%s], "\
               "recvBuf[%p], recvType[%s], streamId[%d], deviceLogicId[%d]",
               tag, sendBuf, sendCountMatrixHash, GetDataTypeEnumStr(sendType).c_str(), recvBuf,
               GetDataTypeEnumStr(recvType).c_str(), streamId, deviceLogicId);

    /* 记录cclBufferSize用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_ALLTOALLVC,
        tag, 0, HCCL_DATA_TYPE_RESERVED, hcclComm->GetConfigInCCLbufferSize(), 0, nullptr,
        hcclComm->GetRankTableCrc()));
    HcclWorkflowMode mode = GetWorkflowMode();
    CHK_PRT_RET(mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_RESERVED, HCCL_ERROR("Invalid Workflow Mode[%d]", mode),
        HCCL_E_INTERNAL);
    HCCL_PROFILER_ADD_TAG(tag, hcclComm->GetIdentifier(), mode);
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, 0, sendBuf, recvBuf, sendType, INVALID_VALUE_RANKID, hcclComm->GetIdentifier(), \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(hcclComm->GetIdentifier(), rankSize, userRank);
    /* 入参的正确性由HCCL确保 */
    CHK_RET(hcclComm->AlltoAllVC(sendBuf, sendCountMatrix, sendType, recvBuf, recvType, stream, tag));

    u64 sendCount = 0;
    for (u32 i = 0; i < rankSize; i++) {
        sendCount += *(static_cast<const u64 *>(sendCountMatrix) + userRank * rankSize + i);
    }
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(hcclComm->GetIdentifier());
    CHK_RET(CallMsprofReportHostApi(hcclComm, HcclCMDType::HCCL_CMD_ALLTOALLVC, beginTime, sendCount, sendType));
    /* 关键状态记录 */
    HcclUs endut = TIME_NOW();
    HCCL_RUN_INFO("HcomAlltoAllVC success, take time [%lld]us, tag[%s], sendBuf[%p], recvBuf[%p], sendCountMatrix[%p], "\
               "sendType[%s], recvType[%s], streamId[%d]", DURATION_US(endut - startut),
               tag, sendBuf, recvBuf, sendCountMatrix, GetDataTypeEnumStr(sendType).c_str(),
               GetDataTypeEnumStr(recvType).c_str(), streamId);
    return HCCL_SUCCESS;
}

HcclResult HcomUnloadTask(const std::string group, const char *tag)
{
    std::shared_ptr<hcclComm> hcclComm;
    if (HcomGetCommByGroup(group.c_str(), hcclComm) == HCCL_SUCCESS) {
        CHK_PRT_RET(hcclComm == nullptr, HCCL_WARNING("[UnloadAllTask]hcclComm is null, "\
        "please check if the initialize process is called."), HCCL_SUCCESS);
        HCCL_INFO("[UnloadTask]HcomUnloadTask: tag[%s]", tag);
        CHK_RET(hcclComm->ClearOpResource(tag));
    } else {
        std::shared_ptr<HcclCommBase> hcclCommbase;
        CHK_PRT_RET(HcpuGetCommByGroup(group.c_str(), hcclCommbase) != HCCL_SUCCESS,
            HCCL_WARNING("[HcomUnloadTask]group[%s] is not exist, do nothing.", group.c_str()), HCCL_SUCCESS);
        CHK_PRT_RET(hcclCommbase == nullptr, HCCL_WARNING("[UnloadAllTask]hcclComm is null, "\
        "please check if the initialize process is called."), HCCL_SUCCESS);
        HCCL_INFO("[UnloadTask]HcomUnloadTask: tag[%s]", tag);
        CHK_RET(hcclCommbase->ClearOpResource(tag));
    }

    return HCCL_SUCCESS;
}

HcclResult HcomGetServerNumAndDeviceNumPerServer(u32 &serverNum, u32 &deviceNumPerServer, u32 &deviceNumPerAggregation)
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    CHK_PRT_RET(hcomInfo.pComm == nullptr,
        HCCL_ERROR("[GetServerNumAndDeviceNumPerServer]hcomInfo.pComm is null, "\
            "please check if the initialize process is called."), HCCL_E_INTERNAL);
    u32 totalDevNum;
    serverNum = hcomInfo.rankTable.serverNum;
    totalDevNum = hcomInfo.rankTable.deviceNum;
    if ((totalDevNum % serverNum) != 0) {
        HCCL_ERROR("devicenum is not Integer.");
    }
    deviceNumPerServer = totalDevNum / serverNum;

    CHK_RET(hcomInfo.pComm->GetDeviceNumPerAggregation(deviceNumPerAggregation));

    return HCCL_SUCCESS;
}

static RankTable_t g_rankTableSetInfo;
HcclResult HcomSetRankTableImpl(const char *rankTableStr)
{
    HCCL_RUN_INFO("Entry-HcomSetRankTable: rankTable \"%s\"", rankTableStr);
    u32 rankTableSize = 0;
    HcclResult ret = HcomCheckRankTable(rankTableStr, rankTableSize);
    CHK_PRT_RET(ret != HCCL_SUCCESS,
        HCCL_ERROR("[Init][Result]errNo[0x%016llx] input rankTable error", HCOM_ERROR_CODE(ret)), ret);

    std::string identify = "0";
    HcomInfo hcomInfo;
    ret = CfgGetClusterInfoWithoutDev(rankTableStr, identify, hcomInfo.params, hcomInfo.rankTable);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Init][Result]errNo[0x%016llx] cfg get ranktable[%p] info error: "\
        "identify[%s]", HCOM_ERROR_CODE(ret), rankTableStr, identify.c_str()), ret);

    g_rankTableSetInfo = hcomInfo.rankTable;
    return HCCL_SUCCESS;
}

HcclResult HcomGetActualRankSizeImpl(const char *group, u32 *rankSize)
{
    (void)group;
    *rankSize = g_rankTableSetInfo.rankNum;
    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphUnloadTask(s64 opBaseHcom, const char *tag)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_PRT_RET(hcclComm == nullptr, HCCL_WARNING("[HcclCommGraphUnloadTask]hcclComm is null, "\
        "please check if the initialize process is called."), HCCL_SUCCESS);
    CHK_RET(SetWorkflowMode(HcclWorkflowMode::HCCL_WORKFLOW_MODE_OPS_KERNEL_INFO_LIB));
    HCCL_INFO("[UnloadTask]HcclCommGraphUnloadTask: tag[%s]", tag);
    CHK_RET(hcclComm->ClearOpResource(tag));
    return HCCL_SUCCESS;
}

HcclResult HcomSetGlobalWorkSpace(const char *group, std::vector<void *> &globalWorkSpaceAddr)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(group, hcclComm));
    CHK_RET(hcclComm->SetGlobalWorkSpace(globalWorkSpaceAddr));
    return HCCL_SUCCESS;
}

HcclResult HcclCommSetGlobalWorkSpace(s64 opBaseHcom, std::vector<void *> &globalWorkSpaceAddr)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(hcclComm->SetGlobalWorkSpace(globalWorkSpaceAddr));
    return HCCL_SUCCESS;
}

HcclResult HcomGetandClearOverFlowTasks(const char *group, std::vector<hccl::HcclDumpInfo> &hcclDumpInfo)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(group, hcclComm));
    CHK_RET(hcclComm->GetandClearOverFlowTasks(hcclDumpInfo));
    return HCCL_SUCCESS;
}

HcclResult HcclCommGetandClearOverFlowTasks(s64 opBaseHcom, std::vector<hccl::HcclDumpInfo> &hcclDumpInfo)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(hcclComm->GetandClearOverFlowTasks(hcclDumpInfo));
    return HCCL_SUCCESS;
}

HcclResult HcomSupportDeterministicOptim(const char *group, bool &isDeterministicOptim)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(group, hcclComm));
    CHK_RET(hcclComm->SupportDeterministicOptim(isDeterministicOptim));
    return HCCL_SUCCESS;
}

HcclResult HcomGetHccsLinkNum(const char *group, u32 &numHccsLink)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(group, hcclComm));
    CHK_RET(hcclComm->GetHccsLinkNum(numHccsLink));
    return HCCL_SUCCESS;
}

HcclResult HcclCommSupportDeterministicOptim(s64 opBaseHcom, bool &isDeterministicOptim)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(hcclComm->SupportDeterministicOptim(isDeterministicOptim));
    return HCCL_SUCCESS;
}

HcclResult HcomAlltoAll(const void *sendBuf, u64 sendCount, HcclDataType sendType,
                        const void *recvBuf, u64 recvCount, HcclDataType recvType,
                        const char *group, rtStream_t stream, const char *tag)
{
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();
    // 入参合法性校验
    CHK_PTR_NULL(sendBuf);
    CHK_PTR_NULL(recvBuf);

    CHK_RET(HcomCheckOpParam(tag, sendCount, sendType, stream));
    CHK_RET(HcomCheckOpParam(tag, recvCount, recvType, stream));
    CHK_RET(HcomCheckDataType(sendType));
    CHK_RET(HcomCheckDataType(recvType));
    s32 streamId = 0;
    CHK_RET(hrtGetStreamId(stream, streamId));
    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    // 根据ranksize校验相关入参
    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    u32 rankSize = 0, rankId = 0;
    CHK_RET(hcclComm->GetRankSize(rankSize));
    CHK_RET(hcclComm->GetUserRank(rankId));

    if (sendBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(sendBuf));
    }
    if (recvBuf != nullptr) {
        CHK_RET(PrintMemoryAttr(recvBuf));
    }

    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomAlltoAll:tag[%s], sendBuf[%p], recvBuf[%p], sendCount[%llu], recvCount[%llu],"
        "sendType[%s], recvType[%s], group[%s], streamId[%d], deviceLogicId[%d]",
        tag, sendBuf, recvBuf, sendCount, recvCount, GetDataTypeEnumStr(sendType).c_str(),
        GetDataTypeEnumStr(recvType).c_str(), strGroup.c_str(), streamId, deviceLogicId);

    /* 记录cclBufferSize用于一致性校验 */
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_ALLTOALLV,
        tag, 0, HCCL_DATA_TYPE_RESERVED, hcclComm->GetConfigInCCLbufferSize(), 0, nullptr,
        hcclComm->GetRankTableCrc()));
    HcclWorkflowMode mode = GetWorkflowMode();
    CHK_PRT_RET(mode == HcclWorkflowMode::HCCL_WORKFLOW_MODE_RESERVED, HCCL_ERROR("Invalid Workflow Mode[%d]", mode),
        HCCL_E_INTERNAL);
    HCCL_PROFILER_ADD_TAG(tag, strGroup, mode);
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());

    HCCL_PROFILER_ADD_OPDATA_OP(tag, sendCount, sendBuf, recvBuf, sendType, INVALID_VALUE_RANKID, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, rankSize, rankId);

    /* 入参的正确性由HCCL确保 */
    CHK_RET(hcclComm->AlltoAll(sendBuf, sendCount, sendType, recvBuf, recvCount, recvType, stream, tag));

    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_ALLTOALLV, beginTime, sendCount, sendType));
    /* 关键状态记录 */
    HcclUs endut = TIME_NOW();
    HCCL_RUN_INFO("HcomAlltoAll success,take time [%lld]us, tag[%s], sendBuf[%p], recvBuf[%p], sendCount[%llu], "\
               "recvCounts[%llu], sendType[%s], recvType[%s], group[%s], streamId[%d]", DURATION_US(endut - startut),
               tag, sendBuf, recvBuf, sendCount, recvCount, GetDataTypeEnumStr(sendType).c_str(),
               GetDataTypeEnumStr(recvType).c_str(), strGroup.c_str(), streamId);
    return HCCL_SUCCESS;
}

HcclResult HcclIgetLookupRequest(HcclComm comm, s32* tag, ServiceHandle* handle, uint64_t* keys, uint64_t keyMaxNum,
    HcclRequest* request)
{
    HCCL_ERROR("[Iget][LookupRequest] is not support HcclIgetLookupRequest interface");
    return HCCL_E_PARA;
}

HcclResult HcomCollRemotePairedParaCheck(const HcomRemoteOperationParams &params)
{
    CHK_PTR_NULL(params.keyAddr);
    CHK_PTR_NULL(params.value);
    CHK_PTR_NULL(params.tableId);
    CHK_PTR_NULL(params.indices);
    CHK_PTR_NULL(params.numUniqued);
    CHK_PTR_NULL(params.psSeg);
    CHK_PTR_NULL(params.psSegNum);

    return HCCL_SUCCESS;
}

HcclResult HcomSetQosCfg(const char *group, const u32 qosCfg)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(group, hcclComm));
    CHK_RET(hcclComm->SetQosCfg(qosCfg));
    return HCCL_SUCCESS;
}

HcclResult HcclCommSetQosCfg(s64 opBaseHcom, const u32 qosCfg)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(hcclComm->SetQosCfg(qosCfg));
    return HCCL_SUCCESS;
}

HcclResult HcomResetQosCfg(const char *group)
{
    std::shared_ptr<hccl::hcclComm> hcclComm;
    CHK_RET(HcomGetCommByGroup(group, hcclComm));
    CHK_RET(hcclComm->ResetQosCfg());
    return HCCL_SUCCESS;
}

HcclResult HcclCommResetQosCfg(s64 opBaseHcom)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(opBaseHcom);
    CHK_RET(hcclComm->ResetQosCfg());
    return HCCL_SUCCESS;
}

HcclResult HcomInitByRankTable(const char *rankTable, uint32_t rankId)
{
    return HcomInitByString(rankTable, std::to_string(rankId).c_str(), HCCL_MODE_SCHED_OS);
}

inline void GenerateHcomSendRecvOpTag(HcomOperationType opType, const char *group, u32 tag, u32 selfRank, u32 peerRank,
    std::string &opTag)
{
    std::string groupStr = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    if (opType == HCOM_OP_TYPE_SEND) {
        opTag = groupStr + "_" + std::to_string(tag) + "_" + std::to_string(selfRank) + "_" + std::to_string(peerRank);
    } else if (opType == HCOM_OP_TYPE_RECV) {
        opTag = groupStr + "_" + std::to_string(tag) + "_" + std::to_string(peerRank) + "_" + std::to_string(selfRank);
    }
    return;
}

HcclResult HcomGather(const char *tag, void *inputPtr, void *outputPtr, u32 rootRank, u64 inputCount,
    HcclDataType dataType, const char *group, rtStream_t stream)
{
    HcclResult ret;
    HcclUs startut = TIME_NOW();
    uint64_t beginTime = hrtMsprofSysCycleTime();

    s32 deviceLogicId = 0;
    CHK_RET(hrtGetDevice(&deviceLogicId));

    CHK_PRT_RET(inputCount == 0, HCCL_WARNING("input count is 0, return gather success"), HCCL_SUCCESS);
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();

    RPT_INPUT_ERR(inputPtr == nullptr, "EI0003", std::vector<std::string>({"ccl_op", "parameter", "value", "tips"}),
        std::vector<std::string>({"HcomGather", "inputPtr", "nullptr", "please check inputPtr"}));
    CHK_PTR_NULL(inputPtr);
    CHK_RET(HcomCheckOpParam(tag, inputCount, dataType, group, stream));

    std::string strGroup = (group == nullptr) ? HCCL_WORLD_GROUP : group;
    s32 streamId = 0;
    ret = hrtGetStreamId(stream, streamId);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Gather][Result]Call hrtGetStreamId error[%d].",
        ret), HCCL_E_RUNTIME);
    /* 接口交互信息日志 */
    HCCL_USER_CRITICAL_LOG("Entry-HcomGather:tag[%s], inputPtr[%p], outputPtr[%p], inputCount[%llu], rootRank[%u], "
        "dataType[%s], group[%s], streamId[%d], deviceLogicId[%d]", tag, inputPtr, outputPtr, inputCount, rootRank,
        GetDataTypeEnumStr(dataType).c_str(), strGroup.c_str(), streamId, deviceLogicId);
    CHK_RET(PrintMemoryAttr(inputPtr));
    if (outputPtr != nullptr) {
        CHK_RET(PrintMemoryAttr(outputPtr));
    }
    std::shared_ptr<hccl::hcclComm> hcclComm = hcomInfo.pComm;
    CHK_RET(HcomGetCommByGroup(strGroup.c_str(), hcclComm));
    CHK_RET(RankConsistentcyChecker::GetInstance().RecordOpPara(HcclCMDType::HCCL_CMD_GATHER, tag, inputCount, dataType,
        rootRank, hcclComm->GetConfigInCCLbufferSize(), 0, strGroup.c_str(), hcclComm->GetRankTableCrc()));
    HCCL_PROFILER_ADD_TAG(tag, strGroup, GetWorkflowMode());
    HCCL_PROFILER_ADD_STREAM_BY_STREAMID(streamId, tag, 0, AlgType::Reserved());
    HCCL_PROFILER_ADD_OPDATA_OP(tag, inputCount, inputPtr, outputPtr, dataType, rootRank, strGroup, \
        HcclReduceOp::HCCL_REDUCE_RESERVED);
    HCCL_PROFILER_ADD_GROUPRANK(strGroup, hcomInfo.params.totalRanks, hcomInfo.params.userRank);

    std::string sTag = tag;
    ret = hcclComm->Gather(sTag, inputPtr, outputPtr, rootRank, inputCount, dataType, stream);
    CHK_PRT_RET(ret != HCCL_SUCCESS, HCCL_ERROR("[Gather][Result]errNo[0x%016llx] hcclComm gather error, "\
        "tag[%s], input_ptr[%p], output_ptr[%p], root_rank[%u], count[%llu], data_type[%s]", HCOM_ERROR_CODE(ret),
        tag, inputPtr, outputPtr, rootRank, inputCount, GetDataTypeEnumStr(dataType).c_str()), ret);

    CHK_RET(RankConsistentcyChecker::GetInstance().DelOpPara(sTag));
    HCCL_PROFILER_DEL_STREAM_BY_STREAMID(streamId);
    HCCL_PROFILER_DEL_TAG(tag);
    HCCL_PROFILER_DEL_OPDATA(tag);
    HCCL_PROFILER_DEL_GROUPRANK(strGroup);
    CHK_RET(CallMsprofReportHostApi(hcclComm.get(), HcclCMDType::HCCL_CMD_GATHER, beginTime, inputCount, dataType));
    /* 关键状态记录 */
    HCCL_RUN_INFO("hcom gather success,take time [%lld]us, tag[%s], input_ptr[%p], output_ptr[%p], root_rank[%u]"\
        "count[%llu], data_type[%s]", DURATION_US(TIME_NOW() - startut), tag, inputPtr, outputPtr, rootRank,
        inputCount, GetDataTypeEnumStr(dataType).c_str());
    return HCCL_SUCCESS;
}

HcclResult HcomGetTopoDesc(const char *group, HcclTopoDescs *topoDescs, uint32_t topoSize)
{
    CHK_PTR_NULL(topoDescs);
    CHK_PTR_NULL(group);

    std::shared_ptr<hcclComm> hcclComm;
    std::shared_ptr<HcclCommBase> hcclCommbase;
    CHK_RET(HcclDeviceRefresh());
    if (HcomGetCommByGroup(group, hcclComm)==HCCL_SUCCESS) {
        CHK_RET(hcclComm->GetTopoDesc(topoDescs, topoSize));
    } else if (HcpuGetCommByGroup(group, hcclCommbase)==HCCL_SUCCESS) {
        CHK_RET(hcclCommbase->GetTopoDesc(topoDescs, topoSize));
    } else {
        return HCCL_E_PTR;
    }

    return HCCL_SUCCESS;
}

bool HcomIsNormalComm(const char *group)
{
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    return (hcomInfo.pComm != nullptr);
}

HcclResult HcclReStartVnic(s64 comm)
{
    // 获取当前world group的通信域
    HcomInfo &hcomInfo = HcomGetCtxHomInfo();
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(comm);
    CHK_PRT_RET(hcclComm == nullptr, HCCL_ERROR("[HcclReStartVnic]hcomInfo.pComm is null"), HCCL_E_PTR);
    CHK_RET(hcclComm->ReStartVnic(hcomInfo.params, hcomInfo.rankTable));
    return HCCL_SUCCESS;
}

HcclResult HcomClearAivSyncBuf(const char *group, bool aivClearEnable)
{
    CHK_PTR_NULL(group);
    std::shared_ptr<hcclComm> hcclComm;
    if (HcomGetCommByGroup(group, hcclComm) == HCCL_SUCCESS) {
        CHK_RET(hcclComm->ClearAivSyncBuf(aivClearEnable));
    }

    return HCCL_SUCCESS;
}

HcclResult HcclCommGraphClearAivSyncBuf(s64 comm, bool aivClearEnable)
{
    hccl::hcclComm* hcclComm = reinterpret_cast<hccl::hcclComm*>(comm);
    if (hcclComm != nullptr) {
        CHK_RET(hcclComm->ClearAivSyncBuf(aivClearEnable));
    }
    return HCCL_SUCCESS;
}

#if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
HcclResult HcclBuildOneSidedService(std::unique_ptr<IHcclOneSidedService> &service, std::unique_ptr<hccl::HcclSocketManager> &socketManager,
    std::unique_ptr<hccl::NotifyPool> &notifyPool);
#endif

__attribute__((constructor)) void CallBackInit()
{
    HcomGroupCallbackFuncInstall(HcomCreateGroupImplHeterog,
        HcomCallBackGroupIsInitHeterog,
        HcomDestroyGroupImplHeterog,
        HcomDestroyOneDeviceHeterog);
    #if (!defined(HCCD)) && (!defined(CCL_KERNEL_AICPU))
        HcclOneSidedServiceCallbackInstall(HcclBuildOneSidedService);
    #endif
}